test-on-edit

1import type { EngineInterface, On, Register } from 'claude-code'
2
3import { candidates, parents, runner, summary } from './find.ts'
4
5const TIMEOUT_MS = 60_000
6
7// Test files running now: an edit while its test still runs adds nothing
8const running = new Set<string>()
9let isOn = true
10// Files edited this turn, and failures waiting for the next prompt
11const edited = new Set<string>()
12let pending: string | undefined
13
14async function project($: EngineInterface, root: string, rel: string) {
15  let modRoot: string | undefined
16  for (const dir of parents(rel)) {
17    if (await $.fs.exists(`${root}/${dir}/.claude-plugin/plugin.json`).catch(() => false)) {
18      modRoot = dir
19      break
20    }
21  }
22  if (modRoot === undefined && (await $.fs.read(`${root}/${rel}`).catch(() => '')).includes(`'claude-code/testing'`)) modRoot = parents(rel)[0] ?? ''
23  const hasUvLock = await $.fs.exists(`${root}/uv.lock`).catch(() => false)
24  let deps: string[] = []
25  try {
26    const pkg = JSON.parse(await $.fs.read(`${root}/package.json`)) as { dependencies?: object; devDependencies?: object }
27    deps = [...Object.keys(pkg.dependencies ?? {}), ...Object.keys(pkg.devDependencies ?? {})]
28  } catch {
29    // No package.json
30  }
31
32  return { hasUvLock, deps, modRoot }
33}
34
35type Result = { test: string; ok: boolean; note: string }
36
37async function findTest($: EngineInterface, path: string, root: string): Promise<string | undefined> {
38  for (const c of candidates(path, root)) if (await $.fs.exists(c).catch(() => false)) return c
39
40  return undefined
41}
42
43async function runTest($: EngineInterface, test: string, root: string): Promise<Result | undefined> {
44  if (running.has(test)) return undefined
45  running.add(test)
46  const rel = test.startsWith(`${root}/`) ? test.slice(root.length + 1) : test
47  const command = runner(rel, await project($, root, rel))
48  const started = await $.clock.now()
49  try {
50    const r = await $.process.run(['/bin/zsh', '-lc', command], { cwd: root, timeoutMs: TIMEOUT_MS })
51    const secs = Math.round(((await $.clock.now()) - started) / 1000)
52
53    return r.exitCode === 0
54      ? { test: rel, ok: true, note: `${rel} passes (${secs}s)` }
55      : { test: rel, ok: false, note: `${rel} FAILS (exit ${r.exitCode}, \`${command}\`):\n${summary(`${r.stdout}\n${r.stderr}`)}` }
56  } catch (err) {
57    return { test: rel, ok: false, note: `${rel} could not run within ${TIMEOUT_MS / 1000}s (${err instanceof Error ? err.message : String(err)})` }
58  } finally {
59    running.delete(test)
60  }
61}
62
63// <metrics: generated from shared/metrics.ts by shared/sync.sh>
64// Shared mod metrics (schema v2). Source of truth: ~/.claude/mods/shared/metrics.ts.
65// shared/sync.sh inlines it between the metrics markers in each mod's register.tsx (the engine only lets $
66// into functions declared in the same file). Edit here, run sync.sh, never edit an inlined copy.
67//
68// Rows appended to ~/.claude/mods/metrics.jsonl, all with { v: 2, t, mod, event, ver, session, project }:
69//   loaded   once per load (logLoaded, or count() in the mod's session.start hook), at session.start: proves which version of the mod is running
70//   turn     once per turn with activity (logTurns, or count() in the mod's turn.complete hook): { hooks: { <event>: calls } }
71//   <other>  outcome rows from track(), e.g. merge-gate 'verdict'
72// Privacy: never put prompt, command, tool-input or matched-secret text in a row; lengths, kinds and counts only.
73
74const APPEND = 'mkdir -p "$HOME/.claude/mods" && printf "%s\\n" "$1" >> "$HOME/.claude/mods/metrics.jsonl"'
75
76
77const ctx = { ver: '', session: '', project: '' }
78const stats = new Map<string, number>()
79let writeFailed = false
80
81const message = (err: unknown) => (err instanceof Error ? err.message : String(err)).slice(0, 300)
82
83function write($: EngineInterface, mod: string, event: string, data: Record<string, unknown>): void {
84  void (async () => {
85    const t = new Date(await $.clock.now()).toISOString()
86    if (!ctx.project) ctx.project = (await $.session.cwd()).split('/').filter(Boolean).pop() ?? ''
87    const line = JSON.stringify({ v: 2, t, mod, event, ver: ctx.ver, session: ctx.session, project: ctx.project, ...data })
88    const { exitCode, stderr } = await $.process.run(['/bin/sh', '-c', APPEND, 'mod-metrics', line], { timeoutMs: 3000 })
89    if (exitCode !== 0) throw new Error(stderr.trim() || `exit ${exitCode}`)
90  })().catch((err: unknown) => {
91    if (writeFailed) return
92    writeFailed = true
93    $.ui.toast(`${mod}: metrics write failed (${message(err)})`)
94  })
95}
96
97// An outcome row: something the mod decided or did. Fire-and-forget.
98function track($: EngineInterface, mod: string, event: string, data: Record<string, unknown> = {}): void {
99  write($, mod, event, data)
100}
101
102async function loaded($: EngineInterface): Promise<void> {
103  const mod = $.plugin.name
104  ctx.session = await $.session.id().catch(() => '')
105  ctx.project = ''
106  try {
107    const manifest = JSON.parse(await $.fs.read(`${$.plugin.root}/.claude-plugin/plugin.json`)) as { version?: string }
108    ctx.ver = manifest.version ?? ''
109  } catch {
110    ctx.ver = '?'
111  }
112  write($, mod, 'loaded', {})
113}
114
115function flush($: EngineInterface): void {
116  const mod = $.plugin.name
117  if (!stats.size) return
118  const hooks = Object.fromEntries(stats)
119  stats.clear()
120  write($, mod, 'turn', { hooks })
121}
122
123// The engine allows one hook per event per plugin, and counts them by spelling. A mod that hooks
124// session.start or turn.complete itself calls count() in that hook, which writes the row; otherwise
125// it calls the matching helper once in register.
126function logLoaded(on: On): void {
127  on('session.start', async ($, e, next) => {
128    await loaded($)
129
130    return next(e)
131  })
132}
133
134function logTurns(on: On): void {
135  on('turn.complete', async ($, e, next) => {
136    flush($)
137
138    return next(e)
139  })
140}
141
142// First statement of every hook: counts the call into this turn's row. The engine only accepts
143// function literals as hooks, so calls are counted from inside rather than by wrapping.
144// In a session.start hook it also writes the loaded row; in a turn.complete hook it writes the turn row.
145function count($: EngineInterface, event: string): void {
146  stats.set(event, (stats.get(event) ?? 0) + 1)
147  if (event === 'session.start') void loaded($)
148  if (event === 'turn.complete') flush($)
149}
150// </metrics>
151
152// <git: generated from shared/git.ts by shared/sync.sh>
153// The git top level of cwd (the session's folder when omitted); undefined outside a repository or when git fails
154async function repoRoot($: EngineInterface, cwd?: string): Promise<string | undefined> {
155  const r = await $.process.run(['git', 'rev-parse', '--show-toplevel'], { ...(cwd ? { cwd } : {}), timeoutMs: 5000 }).catch(() => undefined)
156
157  return r && r.exitCode === 0 ? r.stdout.trim() || undefined : undefined
158}
159// </git>
160
161export const register: Register = on => {
162  on('session.start', async ($, e, next) => {
163    count($, 'session.start')
164    await $.command.register({ name: 'test-on-edit', description: 'Run the edited file\'s tests after each edit: /test-on-edit off | on' })
165
166    return next(e)
167  })
168
169  on('command.run', { command: 'test-on-edit' }, async ($, e) => {
170    count($, 'command.run{test-on-edit}')
171    const arg = e.args.trim()
172    if (arg === 'off' || arg === 'on') isOn = arg === 'on'
173
174    return { text: `${isOn ? 'on' : 'off for this session'}.` }
175  })
176
177  // During a turn: only note which files changed
178  on('tool.call', async ($, e, next) => {
179    count($, 'tool.call')
180    const ran = await next(e)
181    if (isOn && (e.tool === 'Edit' || e.tool === 'Write') && ran.deny === undefined && !ran.isError) edited.add(String(e.file_path))
182
183    return ran
184  })
185
186  // When Claude's answer ends: run each edited file's tests once, off the turn's path
187  on('turn.complete', async ($, e, next) => {
188    count($, 'turn.complete')
189    const done = await next(e)
190    if (e.agentId !== undefined || edited.size === 0 || !isOn) return done
191    const paths = [...edited]
192    edited.clear()
193    void (async () => {
194      const cwd = await $.session.cwd()
195      const root = (await repoRoot($, cwd)) ?? cwd
196      const tests = new Set<string>()
197      for (const p of paths) {
198        const t = await findTest($, p, root)
199        if (t) tests.add(t)
200      }
201      if (tests.size === 0) return
202      $.ui.status(`test-on-edit: running ${tests.size} test file${tests.size > 1 ? 's' : ''}…`)
203      const results = (await Promise.all([...tests].map(t => runTest($, t, root)))).filter((r): r is Result => !!r)
204      const failed = results.filter(r => !r.ok)
205      track($, 'test-on-edit', 'run', { files: results.length, failed: failed.length })
206      if (failed.length === 0) {
207        pending = undefined
208        $.ui.status(undefined)
209        $.ui.toast(`test-on-edit: ${results.length} test file${results.length > 1 ? 's' : ''} pass`)
210
211        return
212      }
213      pending = `test-on-edit ran the tests for files edited last turn, and some fail:\n\n${failed.map(r => r.note).join('\n\n')}`
214      $.ui.status(`test-on-edit: ${failed.length} failing (${failed.map(r => r.test.split('/').pop()).join(', ')}); Claude sees it with your next prompt`)
215      $.ui.toast(`test-on-edit: ${failed.map(r => r.test).join(', ')} failing`)
216    })().catch(() => $.ui.status(undefined))
217
218    return done
219  })
220
221  // Hand failures to Claude with the next prompt, once
222  on('prompt.submit', async ($, e, next) => {
223    count($, 'prompt.submit')
224    if (!pending || (e.origin.kind !== 'composer' && e.origin.kind !== 'bridge')) return next(e)
225    const note = pending
226    pending = undefined
227    $.ui.status(undefined)
228    track($, 'test-on-edit', 'delivered', { chars: note.length })
229
230    return next({ ...e, context: [...(e.context ?? []), note] })
231  }).catch(($, e, next) => next(e))
232}
233

1// Pure helpers: no `$`, so tests call them directly.
2
3const SOURCE = /\.(m?[jt]sx?|cjs|py)$/
4export const isTestFile = (path: string) => /(\.(test|spec)\.[^/]+$|(^|\/)test_[^/]+\.py$|_test\.py$)/.test(path)
5
6const split = (path: string) => {
7  const slash = path.lastIndexOf('/')
8  const dir = path.slice(0, slash)
9  const file = path.slice(slash + 1)
10  const dot = file.indexOf('.')
11
12  return { dir, stem: dot < 0 ? file : file.slice(0, dot), ext: file.slice(file.lastIndexOf('.') + 1) }
13}
14
15// Where this file's tests usually live, most likely first. `root` is the repo root.
16export function candidates(path: string, root: string): string[] {
17  if (!SOURCE.test(path) || /(^|\/)(node_modules|dist|build|\.next|\.venv)\//.test(path)) return []
18  if (isTestFile(path)) return [path]
19  const { dir, stem, ext } = split(path)
20  if (ext === 'py') return [`${root}/tests/test_${stem}.py`, `${dir}/test_${stem}.py`, `${dir}/tests/test_${stem}.py`, `${dir}/${stem}_test.py`]
21  const exts = [...new Set([ext, 'mjs', 'js', 'ts', 'tsx'])]
22
23  return exts.flatMap(x => [
24    `${dir}/${stem}.test.${x}`,
25    `${dir}/${stem}.spec.${x}`,
26    `${dir}/__tests__/${stem}.test.${x}`,
27    `${root}/test/${stem}.test.${x}`,
28    `${root}/tests/${stem}.test.${x}`,
29  ])
30}
31
32// modRoot: the nearest folder above the test with .claude-plugin/plugin.json (a mod), relative to the repo root
33export type Project = { hasUvLock: boolean; deps: readonly string[]; modRoot?: string }
34
35// Folders from the test's own up to (not including) the repo root, nearest first
36export function parents(test: string): string[] {
37  const parts = test.split('/').slice(0, -1)
38
39  return parts.map((_, i) => parts.slice(0, parts.length - i).join('/'))
40}
41
42// The command that runs one test file in this project
43export function runner(test: string, p: Project): string {
44  const quote = (s: string) => `'${s.replace(/'/g, `'\\''`)}'`
45  const q = quote(test)
46  if (p.modRoot !== undefined) return `claude plugin test ${quote(p.modRoot || '.')}`
47  if (test.endsWith('.py')) return p.hasUvLock ? `uv run pytest -q ${q}` : `python3 -m pytest -q ${q}`
48  if (p.deps.includes('vitest')) return `npx vitest run ${q}`
49  if (p.deps.includes('jest')) return `npx jest ${q}`
50
51  return `node --test ${q}`
52}
53
54export function summary(output: string, max = 25): string {
55  return output.trimEnd().split('\n').slice(-max).join('\n')
56}
57