test-ledger

1// test-ledger: watches the Bash calls Claude makes and keeps the last 40 that are test, lint, build, format or check
2// commands (pytest, R CMD check, devtools::test, cargo, npm/pnpm/yarn scripts, ruff, mypy, eslint, quarto, ...), with
3// pass or fail, time and duration. `/test-ledger` shows them in a pane, newest first, with a Clear button.
4// Memory only: secrets in a command are replaced by *** and the command cut to 200 characters before it is kept.
5// FAIL OPEN: it never blocks or changes a call; if recording fails, the result goes back to the model untouched.
6import { atom, read, update } from 'claude-code'
7import type { EngineInterface, Register } from 'claude-code'
8
9import type { TestLedgerRun } from '../types'
10import { classify, clockTime, cut, exitCodeOf, scrub, seconds } from './lib'
11
12type Dollar = EngineInterface
13
14const COMMAND = 'test-ledger'
15const PANE = 'test-ledger'
16const KEEP = 40
17
18const runs = atom({ plugin: 'test-ledger', key: 'runs' } as const, [])
19const isOff = atom({ plugin: 'test-ledger', key: 'isOff' } as const, false)
20
21let counter = 0
22
23async function setSwitch($: Dollar, off: boolean): Promise<void> {
24  if (off) await $.store.set('off', true)
25  else await $.store.delete('off')
26  await update($, isOff, () => off)
27}
28
29async function isSwitchedOff($: Dollar): Promise<boolean> {
30  try {
31    return await read($, isOff)
32  } catch {
33    return false
34  }
35}
36
37async function status($: Dollar): Promise<string> {
38  const list = await read($, runs)
39  const failed = list.filter(r => r.status === 'fail').length
40  const last = list[list.length - 1]
41  const lines = [`test-ledger · ${(await isSwitchedOff($)) ? 'off' : 'on'} · ${list.length} run${list.length === 1 ? '' : 's'} recorded, ${failed} failed`]
42  if (last !== undefined) lines.push(`Last: ${last.status} · ${last.tool} · ${clockTime(last.at)}`)
43
44  return lines.join('\n')
45}
46
47async function run($: Dollar, args: string): Promise<string> {
48  const word = args.trim().toLowerCase()
49  if (word === 'status') return status($)
50  if (word === 'clear') {
51    await update($, runs, () => [])
52
53    return 'test-ledger · cleared'
54  }
55  if (word === 'on' || word === 'off') {
56    try {
57      await setSwitch($, word === 'off')
58    } catch {
59      return 'test-ledger · that did not work, nothing changed'
60    }
61
62    return word === 'on' ? 'test-ledger · on' : 'test-ledger · off · /test-ledger on records again'
63  }
64  if (word !== '') return `test-ledger · unknown option “${word.slice(0, 30)}”\nUse /test-ledger, or /test-ledger on, off, clear or status.`
65  await $.ui.open({ id: PANE, title: 'Checks', columns: 70 })
66
67  return status($)
68}
69
70async function record($: Dollar, command: string, ran: { isError?: true; text?: string; result?: unknown }, startedAt: number): Promise<void> {
71  if (await isSwitchedOff($)) return
72  const match = classify(command)
73  if (match === undefined) return
74  const at = await $.clock.now()
75  const result = (ran.result ?? {}) as { interrupted?: unknown; backgroundTaskId?: unknown }
76  const isBackground = ran.isError !== true && typeof result.backgroundTaskId === 'string' && result.backgroundTaskId !== ''
77  const status: TestLedgerRun['status'] = ran.isError === true || result.interrupted === true ? 'fail' : isBackground ? 'background' : 'pass'
78  const exitCode = status === 'fail' ? exitCodeOf(ran.text) : undefined
79  counter += 1
80  const entry: TestLedgerRun = {
81    id: `${at}-${counter}`,
82    kind: match.kind,
83    tool: match.tool,
84    command: cut(scrub(command).replace(/\s+/g, ' ').trim(), 200),
85    status,
86    ...(exitCode === undefined ? {} : { exitCode }),
87    at,
88    ms: Math.max(0, at - startedAt),
89  }
90  await update($, runs, list => [...list, entry].slice(-KEEP))
91}
92
93const MARK: Record<TestLedgerRun['status'], { glyph: string; color: string }> = {
94  pass: { glyph: '✓', color: 'green' },
95  fail: { glyph: '✗', color: 'red' },
96  background: { glyph: '…', color: 'gray' },
97}
98
99export const register: Register = on => {
100  on('session.start', async ($, e, next) => {
101    try {
102      const stored = (await $.store.get('off')) === true
103      await update($, isOff, () => stored)
104    } catch {
105      // reads as on
106    }
107    try {
108      await $.command.register({ name: COMMAND, description: 'Show the test, lint and build commands run this session', argumentHint: '[on|off|clear|status]' })
109    } catch {
110      // fail-open: the session goes on without the command
111    }
112
113    return next(e)
114  })
115
116  on('command.run', { command: COMMAND }, async ($, e) => ({ text: await run($, e.args) })).catch(() => ({
117    text: 'test-ledger · could not read its own state',
118  }))
119
120  // Watches only: the call runs as the model made it and its result goes back unchanged.
121  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
122    const startedAt = await $.clock.now().catch(() => 0)
123    const ran = await next(e)
124    if (ran.deny !== undefined) return ran // never ran
125    try {
126      await record($, String(e.command ?? ''), ran, startedAt)
127    } catch {
128      // a ledger problem never reaches the model
129    }
130
131    return ran
132  }).catch(($, e, next) => next(e))
133
134  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
135    const { Box, Button, Text } = $.ui.resolve(e)
136    const list = await read($, runs)
137    const off = await isSwitchedOff($)
138    const failed = list.filter(r => r.status === 'fail').length
139    // Newest first, as many as fit in about 9000 characters.
140    let budget = 9000
141    const shown: TestLedgerRun[] = []
142    for (const r of [...list].reverse()) {
143      budget -= r.command.length + r.tool.length + 60
144      if (budget < 0) break
145      shown.push(r)
146    }
147    const rows = shown.map(r => {
148      const mark = MARK[r.status]
149      const code = r.exitCode === undefined ? '' : ` (exit ${r.exitCode})`
150
151      return (
152        <Box flexDirection="column">
153          <Text wrap="truncate-end">
154            <Text color={mark.color}>{mark.glyph} {r.status}</Text> {r.kind} · {r.tool} · {clockTime(r.at)} · {seconds(r.ms)}{code}
155          </Text>
156          <Text dimColor wrap="truncate-end">
157            {'  '}{r.command}
158          </Text>
159        </Box>
160      )
161    })
162
163    return (
164      <Box flexDirection="column">
165        <Box>
166          <Text bold>
167            {list.length} run{list.length === 1 ? '' : 's'} · {failed} failed{off ? ' · off' : ''}{' '}
168          </Text>
169          <Button key="clear" label="Clear" onPress={async () => void (await update($, runs, () => []))} />
170        </Box>
171        {list.length === 0 && <Text dimColor>No test, lint, build, format or check commands yet.</Text>}
172        {rows}
173        {shown.length < list.length && <Text dimColor>+{list.length - shown.length} older runs not shown</Text>}
174      </Box>
175    )
176  })
177}
178

1// test-ledger: pure helpers (no engine access): which shell commands are checks, and secret scrubbing.
2import type { TestLedgerKind } from '../types'
3
4// ---- secrets ------------------------------------------------------------------------------------------------------
5// The same function is copied into error-inbox (mods cannot import each other); keep the two in step.
6
7const SECRET_PATTERNS: [RegExp, string][] = [
8  [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{10,}/g, '***'],
9  [/\bgithub_pat_[A-Za-z0-9_]{10,}/g, '***'],
10  [/\bsk-[A-Za-z0-9_-]{8,}/g, '***'],
11  [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, '***'],
12  [/\bxox[abprs]-[A-Za-z0-9-]{10,}/g, '***'],
13  [/\bglpat-[A-Za-z0-9_-]{10,}/g, '***'],
14  [/\bAIza[0-9A-Za-z_-]{30,}/g, '***'],
15  [/-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g, '***'],
16  [/\b(Bearer|Basic|Token)(\s+)[A-Za-z0-9._~+/=-]{6,}/gi, '$1$2***'],
17  [/(--?(?:token|password|passwd|pass|secret|api[-_]?key|access[-_]?key|auth|private[-_]?key)(?:=|\s+))("[^"]*"|'[^']*'|\S+)/gi, '$1***'],
18  [/((?:password|passwd|pwd|secret|token|api[-_]?key|apikey|access[-_]?key|auth[-_]?key|client[-_]?secret)["']?\s*[=:]\s*)("[^"]*"|'[^']*'|[^\s&;,'"]+)/gi, '$1***'],
19  [/(\b[a-z][a-z0-9+.-]*:\/\/[^\s/:@]+:)[^\s/@]+@/gi, '$1***@'],
20]
21
22/** Replaces the obvious secrets (API tokens, keys, Bearer values, password=..., --token x, URL passwords) with ***. */
23export function scrub(text: string): string {
24  return SECRET_PATTERNS.reduce((out, [pattern, to]) => out.replace(pattern, to), text)
25}
26
27/** Drops control characters (keeps newline and tab) and cuts to `limit` characters with an ellipsis. */
28export function cut(text: string, limit: number): string {
29  const clean = text.replace(/\r\n/g, '\n').replace(/[\u0000-\u0008\u000b-\u001f\u007f-\u009f]/g, '')
30
31  return clean.length > limit ? `${clean.slice(0, limit - 1)}…` : clean
32}
33
34// ---- which commands are checks -----------------------------------------------------------------------------------
35
36export type Match = { kind: TestLedgerKind; tool: string }
37
38/** Wrappers that run the real command: `uv run pytest`, `npx eslint`, `time cargo test`. */
39// Written as text so the capability audit does not read these words as programs this mod starts (it starts none).
40const WRAPPERS: string[][] = 'uv run|poetry run|pipenv run|pdm run|hatch run|rye run|pixi run|pnpm exec|pnpm dlx|yarn dlx|npm exec|npx|bunx|time|env|command'
41  .split('|')
42  .map(w => w.split(' '))
43
44const base = (word: string): string =>
45  word.replace(/^["']|["']$/g, '').split(/[\\/]/).pop()!.toLowerCase().replace(/\.(exe|cmd|bat|ps1)$/, '')
46
47function unwrap(words: string[]): string[] {
48  let w = words
49  for (let guard = 0; guard < 6; guard++) {
50    while (w.length > 0 && /^[A-Za-z_][A-Za-z0-9_]*=/.test(w[0]!)) w = w.slice(1) // FOO=bar cmd
51    const wrapper = WRAPPERS.find(p => p.every((part, i) => base(w[i] ?? '') === part))
52    if (wrapper === undefined) return w
53    w = w.slice(wrapper.length)
54    while (w.length > 0 && w[0]!.startsWith('-')) w = w.slice(1) // npx -y eslint
55  }
56
57  return w
58}
59
60const PY_MODULES: Record<string, TestLedgerKind> = {
61  pytest: 'test', unittest: 'test', tox: 'test', nox: 'test', mypy: 'check', pyright: 'check',
62  ruff: 'lint', flake8: 'lint', pylint: 'lint', black: 'format', isort: 'format',
63}
64
65const SCRIPT_KINDS: [RegExp, TestLedgerKind][] = [
66  [/^(test|tests|t|unit|e2e|spec)\b/, 'test'],
67  [/^(build|compile|bundle)\b/, 'build'],
68  [/^(lint|eslint)\b/, 'lint'],
69  [/^(format|fmt|prettier)\b/, 'format'],
70  [/^(check|typecheck|type-check|tsc|types)\b/, 'check'],
71]
72
73const scriptKind = (name: string | undefined): TestLedgerKind | undefined =>
74  name === undefined ? undefined : SCRIPT_KINDS.find(([re]) => re.test(name.toLowerCase()))?.[1]
75
76function rCode(code: string): TestLedgerKind | undefined {
77  if (/devtools::test\b|testthat::test_|\btest_(dir|file|local|check|package)\s*\(/.test(code)) return 'test'
78  if (/devtools::check\b|rcmdcheck/.test(code)) return 'check'
79  if (/\blintr\b/.test(code)) return 'lint'
80  if (/\bstyler\b/.test(code)) return 'format'
81  if (/devtools::build\b|pkgbuild::build\b/.test(code)) return 'build'
82
83  return undefined
84}
85
86/** What one simple command is, or undefined when it is no test, lint, build, format or check. */
87export function classifySegment(segment: string, rest = segment): Match | undefined {
88  const words = unwrap(segment.trim().split(/\s+/).filter(Boolean))
89  if (words.length === 0) return undefined
90  const [t0, t1, t2] = [base(words[0]!), words[1]?.toLowerCase(), words[2]?.toLowerCase()]
91
92  switch (t0) {
93    case 'pytest':
94    case 'py.test':
95      return { kind: 'test', tool: 'pytest' }
96    case 'tox':
97    case 'nox':
98      return { kind: 'test', tool: t0 }
99    case 'python':
100    case 'python3':
101    case 'py': {
102      const m = words.indexOf('-m')
103      const mod = m >= 0 ? words[m + 1]?.toLowerCase() : undefined
104      if (mod === undefined || PY_MODULES[mod] === undefined) return undefined
105      if (mod === 'ruff') return { kind: words[m + 2]?.toLowerCase() === 'format' ? 'format' : 'lint', tool: 'ruff' }
106
107      return { kind: PY_MODULES[mod]!, tool: mod }
108    }
109    case 'ruff':
110      return { kind: t1 === 'format' ? 'format' : 'lint', tool: 'ruff' }
111    case 'mypy':
112    case 'pyright':
113      return { kind: 'check', tool: t0 }
114    case 'flake8':
115    case 'pylint':
116      return { kind: 'lint', tool: t0 }
117    case 'black':
118    case 'isort':
119      return { kind: 'format', tool: t0 }
120    case 'rscript':
121    case 'r': {
122      if (t0 === 'r' && t1 === 'cmd') {
123        if (t2 === 'check') return { kind: 'check', tool: 'R CMD check' }
124        if (t2 === 'build') return { kind: 'build', tool: 'R CMD build' }
125
126        return undefined
127      }
128      const kind = rCode(rest) // R code may hold ; or | itself, so read on to the end of the line
129
130      return kind === undefined ? undefined : { kind, tool: t0 === 'r' ? 'R' : 'Rscript' }
131    }
132    case 'air':
133      return t1 === 'format' ? { kind: 'format', tool: 'air' } : undefined
134    case 'jarl':
135      return { kind: 'lint', tool: 'jarl' }
136    case 'cargo': {
137      const kinds: Record<string, TestLedgerKind> = { test: 'test', nextest: 'test', clippy: 'lint', build: 'build', fmt: 'format', check: 'check' }
138      const kind = t1 === undefined ? undefined : kinds[t1]
139
140      return kind === undefined ? undefined : { kind, tool: `cargo ${t1}` }
141    }
142    case 'npm':
143    case 'pnpm':
144    case 'yarn':
145    case 'bun': {
146      if (t1 === 'test' || t1 === 't') return { kind: 'test', tool: `${t0} test` }
147      if (t1 === 'run' || t1 === 'run-script') {
148        const kind = scriptKind(t2)
149
150        return kind === undefined ? undefined : { kind, tool: `${t0} run ${t2}` }
151      }
152      if (t0 !== 'npm') {
153        const kind = scriptKind(t1)
154        if (kind !== undefined) return { kind, tool: `${t0} ${t1}` }
155      }
156
157      return undefined
158    }
159    case 'vitest':
160    case 'jest':
161    case 'mocha':
162      return { kind: 'test', tool: t0 }
163    case 'eslint':
164      return { kind: 'lint', tool: 'eslint' }
165    case 'prettier':
166      return { kind: 'format', tool: 'prettier' }
167    case 'biome': {
168      if (t1 === 'format') return { kind: 'format', tool: 'biome' }
169      if (t1 === 'check' || t1 === 'ci') return { kind: 'check', tool: 'biome' }
170
171      return { kind: 'lint', tool: 'biome' }
172    }
173    case 'tsc':
174      return { kind: 'check', tool: 'tsc' }
175    case 'make': {
176      const kind = scriptKind(t1)
177
178      return kind === undefined ? undefined : { kind, tool: `make ${t1}` }
179    }
180    case 'quarto':
181      if (t1 === 'render') return { kind: 'build', tool: 'quarto render' }
182      if (t1 === 'check') return { kind: 'check', tool: 'quarto check' }
183
184      return undefined
185    case 'go':
186      if (t1 === 'test') return { kind: 'test', tool: 'go test' }
187      if (t1 === 'vet') return { kind: 'lint', tool: 'go vet' }
188      if (t1 === 'build') return { kind: 'build', tool: 'go build' }
189
190      return undefined
191    default:
192      return undefined
193  }
194}
195
196/** The first check in a shell line (split at &&, ||, ;, | and newlines), with the other tools it runs named too. */
197export function classify(command: string): Match | undefined {
198  const found: Match[] = []
199  let start = 0
200  for (const sep of [...command.matchAll(/&&|\|\||[;|\n]/g), undefined]) {
201    const end = sep === undefined ? command.length : sep.index!
202    const match = classifySegment(command.slice(start, end), command.slice(start))
203    if (match !== undefined) found.push(match)
204    if (sep !== undefined) start = end + sep[0].length
205  }
206  if (found.length === 0) return undefined
207  const tools = [...new Set(found.map(m => m.tool))]
208
209  return { kind: found[0]!.kind, tool: tools.slice(0, 3).join(' + ') + (tools.length > 3 ? ' + …' : '') }
210}
211
212/** The exit code a failed Bash result names in its text ("Exit code 2"), when it does. */
213export function exitCodeOf(text: string | undefined): number | undefined {
214  const m = /Exit code (\d+)/.exec(text ?? '')
215
216  return m ? Number(m[1]) : undefined
217}
218
219export const seconds = (ms: number): string => (ms < 1000 ? `${Math.max(0, Math.round(ms))}ms` : ms < 60_000 ? `${(ms / 1000).toFixed(1)}s` : `${Math.floor(ms / 60_000)}m${String(Math.round((ms % 60_000) / 1000)).padStart(2, '0')}s`)
220
221export const clockTime = (ms: number): string => {
222  const d = new Date(ms)
223
224  return [d.getHours(), d.getMinutes(), d.getSeconds()].map(n => String(n).padStart(2, '0')).join(':')
225}
226

1export type TestLedgerKind = 'test' | 'lint' | 'build' | 'format' | 'check'
2
3/** One finished check command Claude ran through Bash. */
4export type TestLedgerRun = {
5  /** Unique within the session. */
6  id: string
7  kind: TestLedgerKind
8  /** The tool(s) recognised: 'pytest', 'cargo clippy', 'Rscript', 'npm run build + eslint'. */
9  tool: string
10  /** The command, secrets replaced by ***, cut to 200 characters. */
11  command: string
12  /** pass: the Bash call succeeded; fail: it returned an error (non-zero exit) or was interrupted; background: moved to the background, outcome unknown. */
13  status: 'pass' | 'fail' | 'background'
14  /** The exit code when the failed result names one. */
15  exitCode?: number
16  /** When it ended, ms since the epoch. */
17  at: number
18  /** How long the call took, ms. */
19  ms: number
20}
21
22declare module 'claude-code' {
23  interface PluginState {
24    'test-ledger': {
25      /** The last 40 runs, oldest first. Memory only: gone when the session ends. */
26      runs: TestLedgerRun[]
27      /** The persisted on/off switch (`$.store` 'off'), mirrored so the moddeck side bar can read it. */
28      isOff: boolean
29    }
30  }
31}
32