auto-test

1import { atom, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { AutoTestLast } from '../types'
5import { detect, fmtSeconds, fromText, parseArgs, tailOf, timeoutOf, whyOf } from './lib'
6import type { TestCommand } from './lib'
7
8type Dollar = EngineInterface
9
10const COMMAND = 'auto-test'
11const EDIT_TOOLS = new Set(['Edit', 'Write', 'NotebookEdit', 'MultiEdit'])
12const HELP = 'auto-test · /auto-test on | off | run | status | command <cmd> (command reset goes back to detecting)'
13
14/** True while auto-test is off. Ships off, so the default is true; the persisted switch is `$.store` 'on'. */
15const isOff = atom({ plugin: 'auto-test', key: 'isOff' } as const, true)
16const last = atom({ plugin: 'auto-test', key: 'last' } as const, 'idle')
17
18type Cfg = { isEnabled: boolean; command: string; timeoutMs: number; tellModel: boolean }
19type LastRun = { status: AutoTestLast; label: string; at: number; ms: number; note: string }
20
21type Ctx = { cfg: Cfg; isDirty: boolean; isRunning: boolean; isRerun: boolean; hasSaid: boolean; lastRun: LastRun | undefined }
22
23/** Runs one call of `$` and swallows its failure: a missing toast or log line must never break a run (fail-open). */
24async function quiet(work: () => unknown): Promise<void> {
25  try {
26    await work()
27  } catch {
28    /* nothing to do */
29  }
30}
31
32function setLast(status: AutoTestLast, $: Dollar) {
33  return update($, last, () => status).catch(() => undefined)
34}
35
36async function isOn(c: Ctx, $: Dollar): Promise<boolean> {
37  if (!c.cfg.isEnabled) return false
38  try {
39    return (await $.store.get('on')) === true
40  } catch {
41    return false
42  }
43}
44
45async function cwdOf(c: Ctx, $: Dollar): Promise<string> {
46  try {
47    return await $.session.cwd()
48  } catch {
49    return ''
50  }
51}
52
53async function projectCommand(c: Ctx, $: Dollar, cwd: string): Promise<string> {
54  try {
55    const v = await $.store.get(`command:${cwd}`)
56    return typeof v === 'string' ? v : ''
57  } catch {
58    return ''
59  }
60}
61
62/** Project command, then the setting, then what the project root looks like. */
63async function commandFor(c: Ctx, $: Dollar, cwd: string): Promise<TestCommand | undefined> {
64  const stored = await projectCommand(c, $, cwd)
65  if (stored !== '') return fromText(stored, 'project')
66  if (c.cfg.command !== '') return fromText(c.cfg.command, 'setting')
67  const at = (f: string) => (cwd === '' ? f : `${cwd.replace(/[\\/]$/, '')}/${f}`)
68  return detect({ exists: f => $.fs.exists(at(f)), read: async f => String(await $.fs.read(at(f))) })
69}
70
71/** Says "could not run" once; later ones stay quiet until a run works. */
72async function couldNot(c: Ctx, $: Dollar, why: string, label: string, started: number) {
73  const now = await $.clock.now()
74  c.lastRun = { status: 'idle', label, at: now, ms: now - started, note: `could not run: ${why}` }
75  await setLast('idle', $)
76  if (c.hasSaid) return
77  c.hasSaid = true
78  await quiet(() => $.ui.toast(`auto-test · could not run: ${why}`))
79}
80
81async function execute(c: Ctx, $: Dollar): Promise<void> {
82  const cwd = await cwdOf(c, $)
83  const started = await $.clock.now()
84  let cmd: TestCommand | undefined
85  try {
86    cmd = await commandFor(c, $, cwd)
87  } catch {
88    cmd = undefined
89  }
90  if (cmd === undefined) return couldNot(c, $, 'no test command found (set one with /auto-test command <cmd>)', '', started)
91
92  await setLast('running', $)
93  let out: { exitCode: number; stdout: string; stderr: string }
94  try {
95    out = await $.process.run(cmd.argv, { cwd: cwd === '' ? undefined : cwd, timeoutMs: c.cfg.timeoutMs })
96  } catch (error) {
97    return couldNot(c, $, whyOf(error, c.cfg.timeoutMs), cmd.label, started)
98  }
99  const ms = (await $.clock.now()) - started
100  // 127 = command not found, 5 from pytest = nothing collected: neither is a verdict on the code.
101  if (out.exitCode === 127) return couldNot(c, $, `${cmd.argv[0] === 'sh' ? 'command' : cmd.argv[0]} not found`, cmd.label, started)
102  if (cmd.label === 'pytest' && out.exitCode === 5) return couldNot(c, $, 'pytest found no tests', cmd.label, started)
103
104  c.hasSaid = false
105  if (out.exitCode === 0) {
106    c.lastRun = { status: 'pass', label: cmd.label, at: started, ms, note: '' }
107    await setLast('pass', $)
108    await quiet(() => $.ui.toast(`auto-test · passed (${fmtSeconds(ms)})`))
109    return
110  }
111
112  const tail = tailOf(out.stdout, out.stderr)
113  c.lastRun = { status: 'fail', label: cmd.label, at: started, ms, note: tail }
114  await setLast('fail', $)
115  await quiet(() => $.ui.toast('auto-test · FAILED'))
116  // One `$.ui.log` call per line: a newline inside one call draws as a stray glyph, and the tail is meant to be copied.
117  await quiet(() => $.ui.log(`auto-test · FAILED · ${cmd.label} · exit ${out.exitCode} · ${fmtSeconds(ms)}`))
118  for (const line of tail.split('\n')) if (line.trim() !== '') await quiet(() => $.ui.log(line))
119  if (c.cfg.tellModel) {
120    const note = `auto-test: the project's tests failed after your last edits (${cmd.label}, exit ${out.exitCode}). Last output:\n${tail}`
121    await quiet(() => $.session.append({ message: { type: 'user', content: [{ type: 'text', text: note }] } }))
122  }
123}
124
125/** Starts a run from a timer so the hook or command returns at once; one run at a time. */
126function start(c: Ctx, $: Dollar) {
127  if (c.isRunning) {
128    c.isRerun = true
129    return
130  }
131  c.isRunning = true
132  $.clock.after(1, () => {
133    void execute(c, $)
134      .catch(() => setLast('idle', $))
135      .finally(() => {
136        c.isRunning = false
137        if (c.isRerun) {
138          c.isRerun = false
139          start(c, $)
140        }
141      })
142  })
143}
144
145async function status(c: Ctx, $: Dollar): Promise<string> {
146  const isSwitchedOn = await isOn(c, $)
147  const head = !c.cfg.isEnabled ? 'disabled (the enabled setting is false)' : isSwitchedOn ? 'on' : 'off (type /auto-test on)'
148  const cwd = await cwdOf(c, $)
149  let cmd: TestCommand | undefined
150  try {
151    cmd = await commandFor(c, $, cwd)
152  } catch {
153    cmd = undefined
154  }
155  const used = cmd === undefined ? 'none found (set one with /auto-test command <cmd>)' : `${cmd.label} (${cmd.source})`
156  const lines = [`auto-test · ${head}`, `command: ${used}`]
157  if (c.isRunning) lines.push('last: running now')
158  else if (c.lastRun === undefined) lines.push('last: no run yet')
159  else {
160    const when = new Date(c.lastRun.at).toLocaleTimeString()
161    const word = c.lastRun.status === 'pass' ? 'passed' : c.lastRun.status === 'fail' ? 'FAILED' : c.lastRun.note
162    lines.push(`last: ${word}${c.lastRun.status === 'idle' ? '' : ` (${fmtSeconds(c.lastRun.ms)})`} at ${when}`)
163  }
164  return lines.join('\n')
165}
166
167async function flip(c: Ctx, $: Dollar, to: boolean): Promise<string> {
168  if (!c.cfg.isEnabled) return 'auto-test · disabled · the enabled setting is false (change it in /config)'
169  try {
170    if (to) await $.store.set('on', true)
171    else await $.store.delete('on')
172    await update($, isOff, () => !to)
173  } catch {
174    return 'auto-test · that did not work, nothing changed'
175  }
176  if (!to) c.isDirty = false
177  return to ? 'auto-test · on · tests run once after a turn that edits files' : 'auto-test · off'
178}
179
180async function setCommand(c: Ctx, $: Dollar, text: string): Promise<string> {
181  const cwd = await cwdOf(c, $)
182  try {
183    if (text === '') {
184      const shown = (await projectCommand(c, $, cwd)) || c.cfg.command
185      return shown === '' ? 'auto-test · no command set · detecting from the project · /auto-test command <cmd> sets one' : `auto-test · command: ${shown}`
186    }
187    if (text.toLowerCase() === 'reset') {
188      await $.store.delete(`command:${cwd}`)
189      return 'auto-test · command cleared · detecting from the project again'
190    }
191    await $.store.set(`command:${cwd}`, text)
192  } catch {
193    return 'auto-test · that did not work, nothing changed'
194  }
195  return `auto-test · command for this project: ${text}`
196}
197
198export const register: Register = (on, options) => {
199  const c: Ctx = {
200    cfg: {
201      isEnabled: options.enabled !== false,
202      command: typeof options.command === 'string' ? options.command.trim() : '',
203      timeoutMs: timeoutOf(options.timeoutSec) * 1000,
204      tellModel: options.tellModel === true,
205    },
206    // Per module instance: a hot reload starts clean, which only costs one skipped run.
207    isDirty: false,
208    isRunning: false,
209    isRerun: false,
210    hasSaid: false,
211    lastRun: undefined,
212  }
213
214  on('session.start', async ($, e, next) => {
215    try {
216      const stored = (await $.store.get('on')) === true // the switch outlives the session
217      await update($, isOff, () => !stored)
218      await update($, last, () => 'idle')
219    } catch {
220      // the switch then reads as off
221    }
222    try {
223      await $.command.register({
224        name: COMMAND,
225        description: 'Run the project tests after Claude edits files: /auto-test on | off | run | status | command <cmd>',
226        argumentHint: 'on | off | run | status | command <cmd>',
227        immediate: true,
228      })
229    } catch {
230      // fail-open: without the command the mod is simply quiet
231    }
232    return next(e)
233  })
234
235  // Only notes that a file was touched. Observational: it never changes the call or its result.
236  on('tool.call', async ($, e, next) => {
237    const ran = await next(e)
238    if (EDIT_TOOLS.has(String(e.tool)) && ran.deny === undefined && ran.isError !== true) c.isDirty = true
239    return ran
240  }).catch(($, e, next) => next(e))
241
242  // The end of the model's turn (not a subagent's): at most one run, and only if something was edited and the switch is on.
243  on('turn.complete', async ($, e, next) => {
244    try {
245      if (e.agentId === undefined && e.reason === 'answer' && c.isDirty) {
246        c.isDirty = false
247        if (await isOn(c, $)) start(c, $)
248      }
249    } catch {
250      // fail-open: a problem here never touches the turn
251    }
252    return next(e)
253  }).catch(($, e, next) => next(e))
254
255  on('command.run', { command: COMMAND }, async ($, e) => {
256    const p = parseArgs(e.args)
257    switch (p.kind) {
258      case 'on':
259        return { text: await flip(c, $, true) }
260      case 'off':
261        return { text: await flip(c, $, false) }
262      case 'status':
263        return { text: await status(c, $) }
264      case 'command':
265        return { text: await setCommand(c, $, p.text) }
266      case 'run':
267        if (!c.cfg.isEnabled) return { text: 'auto-test · disabled · the enabled setting is false' }
268        start(c, $)
269        return { text: 'auto-test · running · the result shows in a moment' }
270      case 'help':
271        return { text: HELP }
272      default:
273        return { text: `auto-test · "${p.word}" is not a choice · ${HELP}` }
274    }
275  }).catch(() => ({ text: 'auto-test · that did not work, nothing changed' }))
276}
277

1// Pure helpers: which test command a project uses, the tail of its output, small formatters.
2
3export type Probe = {
4  exists: (relPath: string) => Promise<boolean>
5  read: (relPath: string) => Promise<string>
6}
7
8export type TestCommand = {
9  /** The program and its arguments, run with no shell (except an override, which runs through `sh -c`). */
10  argv: string[]
11  /** What the person sees: "pytest", "npm test", or the override text. */
12  label: string
13  /** Where it came from: 'detected', 'setting' or 'project'. */
14  source: 'detected' | 'setting' | 'project'
15}
16
17export const TAIL_LINES = 40
18const TAIL_CHARS = 4000
19
20async function safe<T>(work: () => Promise<T>, fallback: T): Promise<T> {
21  try {
22    return await work()
23  } catch {
24    return fallback
25  }
26}
27
28/** A package.json whose "test" script is a real one (not npm's "no test specified" placeholder). */
29export function hasNpmTest(packageJson: string): boolean {
30  try {
31    const script = (JSON.parse(packageJson) as { scripts?: Record<string, unknown> }).scripts?.test
32    return typeof script === 'string' && script.trim() !== '' && !/no test specified/i.test(script)
33  } catch {
34    return false
35  }
36}
37
38/** The command an override text stands for. */
39export const fromText = (text: string, source: 'setting' | 'project'): TestCommand => ({
40  argv: ['sh', '-c', text],
41  label: text,
42  source,
43})
44
45/** Looks at the project root and names its test command, or undefined when none is recognised. */
46export async function detect(p: Probe): Promise<TestCommand | undefined> {
47  const has = (f: string) => safe(() => p.exists(f), false)
48  const pythonProject =
49    (await has('pytest.ini')) ||
50    (await has('pyproject.toml')) ||
51    ((await has('tests')) && ((await has('setup.py')) || (await has('setup.cfg')) || (await has('requirements.txt'))))
52  if (pythonProject) return { argv: ['python3', '-m', 'pytest', '-q'], label: 'pytest', source: 'detected' }
53  if (await has('package.json')) {
54    if (hasNpmTest(await safe(() => p.read('package.json'), ''))) return { argv: ['npm', 'test'], label: 'npm test', source: 'detected' }
55  }
56  if (await has('Cargo.toml')) return { argv: ['cargo', 'test'], label: 'cargo test', source: 'detected' }
57  if (await has('DESCRIPTION')) return { argv: ['Rscript', '-e', 'devtools::test()'], label: "Rscript -e 'devtools::test()'", source: 'detected' }
58  if (await has('go.mod')) return { argv: ['go', 'test', './...'], label: 'go test ./...', source: 'detected' }
59  return undefined
60}
61
62const ANSI = /\u001b\[[0-9;?]*[ -/]*[@-~]/g
63
64/** The last lines of the output (stdout then stderr), without colour codes, never more than a few thousand characters. */
65export function tailOf(stdout: string, stderr: string, lines: number = TAIL_LINES): string {
66  const all = `${stdout}\n${stderr}`.replace(ANSI, '').replace(/\r/g, '').split('\n')
67  while (all.length > 0 && all[all.length - 1]!.trim() === '') all.pop()
68  let text = all.slice(-lines).join('\n').replace(/^\n+/, '')
69  if (text.length > TAIL_CHARS) text = `…${text.slice(-TAIL_CHARS)}`
70  return text
71}
72
73/** "12s", or "3m 05s" for a long run. */
74export function fmtSeconds(ms: number): string {
75  const s = Math.max(0, Math.round(ms / 1000))
76  return s < 60 ? `${s}s` : `${Math.floor(s / 60)}m ${String(s % 60).padStart(2, '0')}s`
77}
78
79export const timeoutOf = (value: unknown): number => {
80  const n = typeof value === 'number' && Number.isFinite(value) ? value : 120
81  return Math.min(600, Math.max(5, Math.round(n)))
82}
83
84export type Parsed =
85  | { kind: 'on' | 'off' | 'run' | 'status' | 'help' }
86  | { kind: 'command'; text: string }
87  | { kind: 'unknown'; word: string }
88
89export function parseArgs(args: string): Parsed {
90  const t = args.trim()
91  if (t === '') return { kind: 'status' }
92  const [word = '', ...rest] = t.split(/\s+/)
93  const w = word.toLowerCase()
94  if (w === 'on' || w === 'off' || w === 'run' || w === 'status' || w === 'help') return { kind: w }
95  if (w === 'command') return { kind: 'command', text: rest.join(' ').trim() }
96  return { kind: 'unknown', word }
97}
98
99/** A short plain reason from the error a failed run rejected with. */
100export function whyOf(error: unknown, timeoutMs: number): string {
101  const text = error instanceof Error ? error.message : String(error)
102  if (/time/i.test(text)) return `timed out after ${fmtSeconds(timeoutMs)}`
103  return text.split('\n')[0]!.slice(0, 120).trim() || 'it did not start'
104}
105

1/** What the last run came to: idle (none yet), running, pass or fail. */
2export type AutoTestLast = 'idle' | 'running' | 'pass' | 'fail'
3
4declare module 'claude-code' {
5  interface PluginState {
6    'auto-test': {
7      /** True while auto-test is off (it ships off). The persisted switch is `$.store` 'on'; loaded at session.start. */
8      isOff: boolean
9      /** The state of the last run, for the moddeck side bar. */
10      last: AutoTestLast
11    }
12  }
13}
14