flaky-detector

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { FlakyState } from '../types'
5import { EMPTY, flakyList, isTestCommand, record, row } from './flaky'
6import type { Ev } from './flaky'
7
8const PANE = 'flaky-detector'
9const TITLE = 'Flaky tests'
10const EDIT_TOOLS = ['Edit', 'Write', 'NotebookEdit']
11const history = atom({ plugin: 'flaky-detector', key: 'history' } as const, EMPTY)
12
13async function track($: EngineInterface, ev: Ev): Promise<void> {
14  let flaky: string | null = null
15  await update($, history, (s: FlakyState) => {
16    const out = record(s, ev)
17    flaky = out.newlyFlaky
18    return out.state
19  })
20  if (flaky !== null) $.ui.toast(`flaky-detector: \`${flaky}\` failed then passed with no code change`)
21}
22
23export const register: Register = on => {
24  on('session.start', async ($, e, next) => {
25    await $.command.register({ name: 'flaky', description: 'Show tests that failed then passed with no code change (open | close | clear)' })
26
27    return next(e)
28  })
29
30  on('command.run', { command: 'flaky' }, async ($, e) => {
31    const arg = e.args.trim()
32    if (arg === 'close' || arg === 'off') {
33      await $.ui.close({ id: PANE })
34      return { text: 'Flaky tests pane closed.' }
35    }
36    if (arg === 'clear') {
37      await update($, history, () => EMPTY)
38      return { text: 'Flaky history cleared.' }
39    }
40    await $.ui.open({ id: PANE, title: TITLE })
41    const count = flakyList(await read($, history)).length
42
43    return { text: `Flaky tests pane opened (${count} flaky).` }
44  })
45
46  on('tool.call', async ($, e, next) => {
47    const ran = await next(e)
48    if (ran.deny !== undefined) return ran
49    if (EDIT_TOOLS.includes(e.tool)) {
50      if (ran.isError !== true) await track($, { kind: 'edit' })
51    } else if (e.tool === 'Bash' && isTestCommand(e.command)) {
52      await track($, { kind: 'test', cmd: e.command, ok: ran.isError !== true, at: await $.clock.now() })
53    }
54
55    return ran
56  })
57
58  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
59    const { Box, Text } = $.ui.resolve(e)
60    const list = flakyList(await read($, history))
61    const room = Math.max(1, (e.viewport?.rows ?? 24) - 4)
62
63    return (
64      <Box flexDirection="column">
65        <Text bold>{list.length} flaky</Text>
66        {list.length === 0 && <Text dimColor>No flaky tests detected.</Text>}
67        {list.slice(0, room).map(({ cmd, rec }) => (
68          <Text>{row(cmd, rec, e.props.bodyColumns)}</Text>
69        ))}
70      </Box>
71    )
72  })
73}
74

1import type { FlakyState, TestRecord } from '../types'
2
3export const EMPTY: FlakyState = { edits: 0, tests: {} }
4export const CAP = 50
5
6export type Ev = { kind: 'edit' } | { kind: 'test'; cmd: string; ok: boolean; at: number }
7
8const TEST_COMMAND = /(^|[\s;&|(])(npm\s+test|pnpm\s+test|yarn\s+test|vitest|jest|pytest|cargo\s+test|go\s+test|swift\s+test|xcodebuild\s(?:\S+\s)*?test|claude\s+plugin\s+test)(\s|$)/
9const ENV_PREFIX = /^(?:[A-Za-z_][A-Za-z0-9_]*=\S*\s+)+/
10const RTK = /(^|[\s;&|(])rtk\s+/g
11
12// Strips env assignments and `rtk` proxies, collapses whitespace.
13export function normalize(command: string): string {
14  const flat = command.trim().replace(/\s+/g, ' ')
15  return flat.replace(ENV_PREFIX, '').replace(RTK, '$1').replace(ENV_PREFIX, '')
16}
17
18export const isTestCommand = (command: string): boolean => TEST_COMMAND.test(normalize(command))
19
20const fresh = (at: number): TestRecord => ({ fails: 0, passes: 0, failedAtEdit: null, flaky: false, lastSeen: at })
21
22// A test is flaky when it passes after a failure with no edit since that failure.
23export function record(state: FlakyState, ev: Ev): { state: FlakyState; newlyFlaky: string | null } {
24  if (ev.kind === 'edit') return { state: { ...state, edits: state.edits + 1 }, newlyFlaky: null }
25  const cmd = normalize(ev.cmd)
26  const prev = state.tests[cmd] ?? fresh(ev.at)
27  const unchanged = prev.failedAtEdit !== null && prev.failedAtEdit === state.edits
28  const flaky = ev.ok && unchanged
29  const next: TestRecord = {
30    fails: prev.fails + (ev.ok ? 0 : 1),
31    passes: prev.passes + (ev.ok ? 1 : 0),
32    failedAtEdit: ev.ok ? null : state.edits,
33    flaky: prev.flaky || flaky,
34    lastSeen: ev.at,
35  }
36  const tests = { ...state.tests, [cmd]: next }
37  const keys = Object.keys(tests)
38  if (keys.length > CAP) {
39    const oldest = keys.reduce((a, b) => ((tests[a]?.lastSeen ?? 0) <= (tests[b]?.lastSeen ?? 0) ? a : b))
40    delete tests[oldest]
41  }
42
43  return { state: { ...state, tests }, newlyFlaky: flaky && !prev.flaky ? cmd : null }
44}
45
46export function flakyList(state: FlakyState): { cmd: string; rec: TestRecord }[] {
47  return Object.entries(state.tests)
48    .filter(([, rec]) => rec.flaky)
49    .map(([cmd, rec]) => ({ cmd, rec }))
50    .sort((a, b) => b.rec.lastSeen - a.rec.lastSeen)
51}
52
53const clock = (at: number) => {
54  const d = new Date(at)
55  return `${String(d.getHours()).padStart(2, '0')}:${String(d.getMinutes()).padStart(2, '0')}`
56}
57
58// "<command> · <fails> fail / <passes> pass · <hh:mm>", the command cut so the row fits `columns`.
59export function row(cmd: string, rec: TestRecord, columns: number): string {
60  const tail = ` · ${rec.fails} fail / ${rec.passes} pass · ${clock(rec.lastSeen)}`
61  const room = Math.max(8, columns - tail.length)
62  const shown = cmd.length > room ? cmd.slice(0, room - 1) + '…' : cmd
63
64  return shown + tail
65}
66

1export type TestRecord = { fails: number; passes: number; failedAtEdit: number | null; flaky: boolean; lastSeen: number }
2export type FlakyState = { edits: number; tests: Record<string, TestRecord> }
3
4declare module 'claude-code' {
5  interface PluginState {
6    'flaky-detector': { history: FlakyState }
7  }
8}
9