Silent Failure Detector

1import type { Register } from 'claude-code'
2
3// Silent Failure Detector
4//
5// A Bash call that exits non-zero already reaches Claude as an error
6// (`isError: true` on the tool.call result). This mod looks at the other
7// case: the call succeeded as far as the shell is concerned, but the output
8// says something went wrong (failed tests, skipped tests, a traceback, an
9// npm ERR!, a coverage threshold that was not met).
10//
11// Checked against claude-code.d.ts 2.1.289: the Bash result record has
12// `stdout`, `stderr`, `interrupted`, `backgroundTaskId?` and
13// `returnCodeInterpretation?`. It has no `exitCode` field; a non-zero exit
14// shows up as `isError: true` on the tool.call result instead.
15
16type Severity = 'high' | 'warning' | 'info'
17type Mode = 'on' | 'strict' | 'off'
18
19type Rule = {
20  label: string
21  pattern: RegExp
22  severity: Severity
23}
24
25type Finding = {
26  label: string
27  severity: Severity
28  line: string
29}
30
31type Report = {
32  command: string
33  at: number
34  findings: Finding[]
35  masking: string[]
36}
37
38// Counts are matched as 1 or more ([1-9]\d*), so "0 failed" and
39// "0 skipped" in a passing summary are not reported.
40const RULES: Rule[] = [
41  // high: the output says the work did not succeed
42  { label: 'failed tests or errors', pattern: /\b[1-9]\d*\s+(?:tests?\s+|specs?\s+|suites?\s+)?(?:failed|failing|failures?|errors?)\b/i, severity: 'high' },
43  { label: 'FAIL / FAILED marker', pattern: /^\s*(?:FAIL|FAILED)\b|\bFAILED\b/m, severity: 'high' },
44  { label: 'npm error', pattern: /^\s*npm (?:ERR!|error)\s/m, severity: 'high' },
45  { label: 'Python traceback', pattern: /Traceback \(most recent call last\)/, severity: 'high' },
46  { label: 'unhandled rejection or exception', pattern: /\bunhandled\s*(?:promise\s*)?(?:rejection|exception|error)\b/i, severity: 'high' },
47  { label: 'error line', pattern: /^\s*(?:Error|TypeError|ReferenceError|SyntaxError|RangeError|ModuleNotFoundError|ImportError|AssertionError)\b[:\s]/m, severity: 'high' },
48  { label: 'failed to load/compile/build', pattern: /\bfailed to (?:load|compile|build|connect|start|resolve|install|fetch)\b/i, severity: 'high' },
49  { label: 'command or file not found', pattern: /\bcommand not found\b|\bNo such file or directory\b|\bPermission denied\b/i, severity: 'high' },
50  { label: 'crash', pattern: /\bsegmentation fault\b|\bcore dumped\b|\bpanicked at\b|\bfatal error\b/i, severity: 'high' },
51
52  // warning: the run may not have checked what it claims to
53  { label: 'tests skipped or pending', pattern: /\b[1-9]\d*\s+(?:tests?\s+)?(?:skipped|pending|todo|xfail(?:ed)?)\b/i, severity: 'warning' },
54  { label: 'no tests ran', pattern: /\bno tests? (?:found|ran|were found|to run|collected)\b|\bRan 0 tests\b|\bcollected 0 items\b/i, severity: 'warning' },
55  { label: 'coverage threshold not met', pattern: /coverage[^\n]*threshold[^\n]*(?:not met|failed)|threshold[^\n]*not met/i, severity: 'warning' },
56  { label: 'exception mentioned', pattern: /\bexception\b/i, severity: 'warning' },
57
58  // info: reported only in strict mode
59  { label: 'warning', pattern: /\bwarning\b|^\s*WARN\b/im, severity: 'info' },
60  { label: 'deprecated API', pattern: /\bdeprecat(?:ed|ion)\b/i, severity: 'info' },
61]
62
63// Shell constructs that hide a failing exit code.
64const MASKS: { label: string; pattern: RegExp }[] = [
65  { label: '`|| true` swallows the exit code', pattern: /\|\|\s*(?:true|:)\b/ },
66  { label: '`; true` / `; exit 0` overrides the exit code', pattern: /;\s*(?:true|exit\s+0)\s*$/ },
67  { label: 'a pipe reports the last command\'s exit code', pattern: /\|\s*(?:tail|head|tee|grep|less|cat|sed|awk|sort|uniq|wc)\b/ },
68]
69
70const MAX_SCAN = 200_000
71const MAX_LINE = 160
72
73function scan(text: string, mode: Mode): Finding[] {
74  const body = text.length > MAX_SCAN ? text.slice(-MAX_SCAN) : text
75  const findings: Finding[] = []
76  for (const rule of RULES) {
77    if (rule.severity === 'info' && mode !== 'strict') continue
78    const match = rule.pattern.exec(body)
79    if (!match) continue
80    findings.push({ label: rule.label, severity: rule.severity, line: lineAround(body, match.index) })
81  }
82  return findings
83}
84
85function lineAround(text: string, index: number): string {
86  const start = text.lastIndexOf('\n', index) + 1
87  const endAt = text.indexOf('\n', index)
88  const line = text.slice(start, endAt === -1 ? undefined : endAt).trim()
89  return line.length > MAX_LINE ? line.slice(0, MAX_LINE - 1) + '…' : line
90}
91
92function masksIn(command: string): string[] {
93  return MASKS.filter(m => m.pattern.test(command)).map(m => m.label)
94}
95
96function shorten(command: string, max = 60): string {
97  const one = command.replace(/\s+/g, ' ').trim()
98  return one.length > max ? one.slice(0, max - 1) + '…' : one
99}
100
101function formatReport(report: Report): string {
102  const lines = [
103    '⚠ POSSIBLE SILENT FAILURE',
104    `Command: ${shorten(report.command, 120)}`,
105    'Exit code: 0 (the tool did not report an error)',
106    'Detected:',
107    ...report.findings.map(f => `  • [${f.severity}] ${f.label}: ${f.line}`),
108  ]
109  if (report.masking.length > 0) {
110    lines.push('Exit code may be masked:', ...report.masking.map(m => `  • ${m}`))
111  }
112  return lines.join('\n')
113}
114
115function contextFor(report: Report, mode: Mode): string | undefined {
116  const relevant = mode === 'strict' ? report.findings : report.findings.filter(f => f.severity === 'high')
117  if (relevant.length === 0) return undefined
118  const signs = relevant.map(f => `- ${f.label}: "${f.line}"`).join('\n')
119  const mask = report.masking.length > 0 ? `\nThe command may hide its exit code (${report.masking.join('; ')}).` : ''
120  return (
121    'silent-failure: the Bash command exited 0, but its output shows signs of failure:\n' +
122    signs +
123    mask +
124    '\nCheck these before you report this step as successful, or say why they do not matter.'
125  )
126}
127
128function isMode(value: unknown): value is Mode {
129  return value === 'on' || value === 'strict' || value === 'off'
130}
131
132const HELP = [
133  '/silent-failure          show the mode and the last finding',
134  '/silent-failure last     show the full last report (alias: inspect)',
135  '/silent-failure strict   also flag warnings and deprecations; tell Claude about every finding',
136  '/silent-failure on       default: flag failures and skipped tests; tell Claude about failures',
137  '/silent-failure off      stop checking',
138  '/silent-failure clear    clear the last report and the status line',
139].join('\n')
140
141export const register: Register = on => {
142  // Module variables reset on each hot reload; $.store keeps them across.
143  let mode: Mode = 'on'
144  let last: Report | undefined
145
146  on('session.start', async ($, e, next) => {
147    const storedMode = await $.store.get('mode')
148    if (isMode(storedMode)) mode = storedMode
149    const storedLast = await $.store.get('last')
150    if (storedLast && typeof storedLast === 'object') last = storedLast as Report
151    await $.command.register({
152      name: 'silent-failure',
153      description: 'Inspect or configure detection of commands that exit 0 but failed',
154      argumentHint: '[last|strict|on|off|clear]',
155    })
156    return next(e)
157  })
158
159  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
160    const ran = await next(e)
161    if (mode === 'off') return ran
162    // A deny or a non-zero exit already reaches Claude as an error.
163    if (ran.deny !== undefined || ran.isError === true) return ran
164
165    const out = ran.result
166    // Interrupted or backgrounded output is incomplete; judging it would mislead.
167    if (out.interrupted || out.backgroundTaskId !== undefined) return ran
168
169    const text = [out.stdout, out.stderr].filter(part => typeof part === 'string' && part.length > 0).join('\n')
170    if (text.length === 0) return ran
171
172    const findings = scan(text, mode)
173    if (findings.length === 0) {
174      $.ui.status(undefined)
175      return ran
176    }
177
178    const report: Report = {
179      command: e.command,
180      at: await $.clock.now(),
181      findings,
182      masking: masksIn(e.command),
183    }
184    last = report
185    await $.store.set('last', report)
186
187    const high = findings.filter(f => f.severity === 'high').length
188    const top = findings[0]?.label ?? 'suspicious output'
189    $.ui.status(`⚠ exit 0 but ${high > 0 ? 'failure' : 'warning'}: ${top} · /silent-failure last`)
190    $.ui.log(`${shorten(e.command)} exited 0 but shows ${findings.map(f => f.label).join(', ')}`)
191
192    const note = contextFor(report, mode)
193    if (note === undefined) return ran
194    return { ...ran, context: [...(ran.context ?? []), note] }
195  })
196
197  on('command.run', { command: 'silent-failure' }, async ($, e) => {
198    const sub = e.args.trim().split(/\s+/)[0]?.toLowerCase() ?? ''
199
200    if (sub === 'strict' || sub === 'on' || sub === 'off') {
201      mode = sub
202      await $.store.set('mode', mode)
203      if (mode === 'off') $.ui.status(undefined)
204      return { text: `mode: ${mode}` }
205    }
206
207    if (sub === 'clear') {
208      last = undefined
209      await $.store.delete('last')
210      $.ui.status(undefined)
211      return { text: 'last report cleared' }
212    }
213
214    if (sub === 'last' || sub === 'inspect') {
215      return { text: last ? formatReport(last) : 'No suspicious success detected yet.' }
216    }
217
218    if (sub === '' ) {
219      const summary = last
220        ? `last: ${shorten(last.command)} → ${last.findings.map(f => f.label).join(', ')}`
221        : 'last: none'
222      return { text: `mode: ${mode}\n${summary}\n\n${HELP}` }
223    }
224
225    return { text: `Unknown option "${sub}".\n\n${HELP}` }
226  })
227}
228