testing

1// browser-test status: a band above the prompt and a toast per failure, for the newest /browser-test
2// run in this project. Everything is read from {directory}/tmp/run-N/run.db, which the skill's runners and
3// orchestrator write as they go; no tool calls are watched.
4import { atom, read, update } from 'claude-code'
5import type { EngineInterface, Register, Timer } from 'claude-code'
6
7import type { RunData, Seen, Snapshot } from '../types'
8import { ago, llmColor, QUERY, STALL_MS, view } from './status'
9
10const POLL_MS = 3000
11
12const snapshot = atom({ plugin: 'testing', key: 'snapshot' } as const, null)
13const seen = atom({ plugin: 'testing', key: 'seen' } as const, null)
14
15let timer: Timer | undefined
16// The db and its WAL's mtimes at the last read: unchanged means the same rows.
17let lastKey = ''
18
19async function newestRun($: EngineInterface, root: string): Promise<{ run: string; dir: string } | null> {
20  const config = `${root}/.browser-tests.json`
21  if (!(await $.fs.exists(config))) return null
22  const directory = (JSON.parse(await $.fs.read(config)) as { directory?: string }).directory
23  if (!directory) return null
24  const tmp = `${root}/${directory}/tmp`
25  if (!(await $.fs.exists(tmp))) return null
26  const runs = (await $.fs.list(tmp))
27    .map(entry => /^run-(\d+)$/.exec(entry.name))
28    .filter((m): m is RegExpExecArray => m !== null)
29    .sort((a, b) => Number(b[1]) - Number(a[1]))
30  for (const m of runs) {
31    const dir = `${directory}/tmp/${m[0]}`
32    if (await $.fs.exists(`${root}/${dir}/run.db`)) return { run: m[0], dir }
33  }
34  return null
35}
36
37async function mtime($: EngineInterface, path: string): Promise<number> {
38  return (await $.fs.exists(path)) ? (await $.fs.stat(path)).mtimeMs : 0
39}
40
41async function toastFailures($: EngineInterface, dir: string, data: RunData) {
42  const top = data.failures.reduce((max, f) => Math.max(max, f.id), 0)
43  const before: Seen | null = await read($, seen)
44  if (before?.dir !== dir) {
45    // First look at this run: what already failed is in the band, not news.
46    await update($, seen, () => ({ dir, failId: top }))
47    return
48  }
49  const fresh = data.failures.filter(f => f.id > before.failId).sort((a, b) => a.id - b.id)
50  for (const f of fresh.slice(-3)) {
51    $.ui.toast(`✗ ${f.scenario}: ${f.reason ?? f.step}`)
52  }
53  if (fresh.length) await update($, seen, () => ({ dir, failId: top }))
54}
55
56async function tick($: EngineInterface) {
57  const root = await $.session.root()
58  const found = await newestRun($, root)
59  const now = await $.clock.now()
60
61  if (!found) {
62    lastKey = ''
63    if (await read($, snapshot)) await update($, snapshot, () => null)
64    return
65  }
66
67  const db = `${root}/${found.dir}/run.db`
68  const key = `${found.dir}:${await mtime($, db)}:${await mtime($, `${db}-wal`)}`
69  const prev = await read($, snapshot)
70
71  if (key === lastKey && prev) {
72    // Same rows. Only redraw while something is unfinished, so "running" can turn into "stalled".
73    const v = view(prev, now)
74    if (v && !v.isDone && v.isVisible) await update($, snapshot, s => (s ? { ...s, checkedAt: now } : s))
75    return
76  }
77
78  let next: Snapshot
79  try {
80    const res = await $.process.run(['sqlite3', db], { stdin: QUERY, timeoutMs: 10_000 })
81    if (res.exitCode !== 0) throw new Error(res.stderr.trim() || `sqlite3 exited ${res.exitCode}`)
82    const data = JSON.parse(res.stdout) as RunData
83    next = { ...found, data, error: null, checkedAt: now }
84    await toastFailures($, found.dir, data)
85    lastKey = key
86  } catch (err) {
87    // A db mid-creation or from an older skill: keep the last good rows, say why, try again next tick.
88    next = { ...found, data: prev?.dir === found.dir ? prev.data : null, error: String(err), checkedAt: now }
89  }
90  await update($, snapshot, () => next)
91}
92
93export const register: Register = on => {
94  timer = undefined
95  lastKey = ''
96
97  on('session.start', async ($, e, next) => {
98    timer?.cancel()
99    await tick($).catch(() => {})
100    timer = $.clock.every(POLL_MS, () => void tick($).catch(() => {}))
101    return next(e)
102  })
103
104  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
105    const snap = await read($, snapshot)
106    const v = snap && view(snap, await $.clock.now())
107    if (!v || !v.isVisible || e.props.hasSurvey) return next(e)
108
109    const { Box, Text } = $.ui.resolve(e)
110    const { counts } = v
111    const waiting = counts.queued + counts.not_run
112
113    return (
114      <Box>
115        <Text>🧪 </Text>
116        <Text color="cyan" bold>browser-test </Text>
117        <Text dimColor>{v.run} · </Text>
118        <Text>{v.phase}</Text>
119        {v.pass && <Text dimColor> ({v.pass})</Text>}
120        <Text dimColor> · {v.steps.done}/{v.steps.total} steps · </Text>
121        <Text color="green">✓{counts.passed} </Text>
122        <Text color={counts.failed ? 'red' : 'gray'}>✗{counts.failed} </Text>
123        {v.running > 0 && <Text color="yellow">▶{v.running} </Text>}
124        {v.stalled > 0 && <Text color="magenta">⏸{v.stalled} stalled </Text>}
125        {counts.incomplete > 0 && <Text color="magenta">◐{counts.incomplete} </Text>}
126        {waiting > 0 && <Text dimColor>· {waiting} waiting </Text>}
127        {v.llmPct !== null && (
128          <Text color={llmColor(v.llmPct)} dimColor={!llmColor(v.llmPct)}>
129            · llm {v.llmPct}%{' '}
130          </Text>
131        )}
132        {v.findings > 0 && <Text color="yellow">· {v.findings} findings </Text>}
133        {v.flaky > 0 && <Text color="magenta">· {v.flaky} flaky </Text>}
134        {v.quietMs > STALL_MS && <Text color="magenta">· quiet {ago(v.quietMs)}</Text>}
135        {snap.error && <Text color="red">· can't read run.db</Text>}
136      </Box>
137    )
138  })
139}
140

1// Everything this mod knows about a /browser-test run comes from its run.db. This file holds the one query
2// and turns its answer into what the band draws. No engine calls, so the test can feed it rows.
3import type { RunData, ScenarioRow, Snapshot } from '../types'
4
5// A runner records after each step. A model-driven step with a screenshot takes well under a minute, so two
6// minutes of silence on an unfinished scenario means it is stuck, not thinking.
7export const STALL_MS = 120_000
8// A run that stopped without marking `done` disappears after this long; a finished one sooner.
9export const IDLE_MS = 30 * 60_000
10export const DONE_MS = 10 * 60_000
11
12// Phases after which an unstarted scenario will not be started.
13const SETTLED = new Set(['report', 'gaps', 'done'])
14
15// Two attempts per scenario. The verdict attempt follows report.js summarize(): the run_id of the scenario's
16// latest functional record. The active attempt is the latest of either pass, so a focus replay counts as work
17// in progress too.
18export const QUERY = `
19WITH
20exp AS (SELECT feature, scenario, MIN(ordinal) AS ord, COUNT(*) AS n FROM expected GROUP BY feature, scenario),
21lastf AS (SELECT feature, scenario, MAX(id) AS mid FROM steps
22           WHERE COALESCE(pass, 'functional') = 'functional' GROUP BY feature, scenario),
23cur AS (SELECT s.feature, s.scenario, s.run_id, COUNT(*) AS rec,
24               SUM(s.status = 'passed') AS passed, SUM(s.status = 'failed') AS failed
25          FROM steps s JOIN lastf l ON s.feature = l.feature AND s.scenario = l.scenario
26         WHERE s.run_id = (SELECT run_id FROM steps WHERE id = l.mid)
27           AND COALESCE(s.pass, 'functional') = 'functional'
28         GROUP BY s.feature, s.scenario),
29lasta AS (SELECT feature, scenario, MAX(id) AS mid FROM steps GROUP BY feature, scenario),
30act AS (SELECT s.feature, s.scenario, s.run_id, COUNT(*) AS rec, SUM(s.status = 'failed') AS failed,
31               MIN(s.ts) AS started, MAX(s.ts) AS last_ts
32          FROM steps s JOIN lasta l ON s.feature = l.feature AND s.scenario = l.scenario
33          JOIN steps m ON m.id = l.mid
34         WHERE s.run_id = m.run_id AND COALESCE(s.pass, 'functional') = COALESCE(m.pass, 'functional')
35         GROUP BY s.feature, s.scenario)
36SELECT json_object(
37  'phase', (SELECT value FROM meta WHERE key = 'phase'),
38  'phase_at', (SELECT CAST(value AS INTEGER) FROM meta WHERE key = 'phase_at'),
39  'initialized_at', (SELECT CAST(value AS INTEGER) FROM meta WHERE key = 'initialized_at'),
40  'last_ts', (SELECT MAX(ts) FROM steps),
41  'scenarios', (SELECT json_group_array(json_object(
42      'feature', e.feature, 'name', e.scenario, 'expected', e.n,
43      'run_id', c.run_id, 'recorded', c.rec, 'passed', c.passed, 'failed', c.failed,
44      'active_run_id', a.run_id, 'active_recorded', a.rec, 'active_failed', a.failed,
45      'active_started', a.started, 'active_ts', a.last_ts))
46    FROM (SELECT * FROM exp ORDER BY feature, ord) e
47    LEFT JOIN cur c ON c.feature = e.feature AND c.scenario = e.scenario
48    LEFT JOIN act a ON a.feature = e.feature AND a.scenario = e.scenario),
49  'lanes', (SELECT json_group_object(lane, c) FROM
50      (SELECT COALESCE(lane, 'unknown') AS lane, COUNT(*) AS c FROM steps
51        WHERE COALESCE(pass, 'functional') = 'functional' GROUP BY 1)),
52  'findings', (SELECT json_group_object(impact, c) FROM
53      (SELECT impact, COUNT(*) AS c FROM findings GROUP BY impact)),
54  'flaky', (SELECT COUNT(*) FROM flags WHERE flaky = 1),
55  'failures', (SELECT json_group_array(json_object('id', id, 'scenario', scenario, 'step', step, 'reason', reason))
56    FROM (SELECT id, scenario, step, reason FROM steps
57           WHERE status = 'failed' AND COALESCE(pass, 'functional') = 'functional'
58           ORDER BY id DESC LIMIT 20))
59);
60`
61
62// What `report summary` would say, plus queued: no record yet in a phase that may still start it.
63export type Verdict = 'passed' | 'failed' | 'incomplete' | 'queued' | 'not_run'
64// What a runner is doing with the scenario right now, if anything.
65export type Activity = 'running' | 'stalled' | null
66
67export type ScenarioView = { row: ScenarioRow; verdict: Verdict; activity: Activity }
68
69export type RunView = {
70  run: string
71  phase: string
72  // run_id of the most recent step: `claude-1`, `repair-1`, `focus-1`
73  pass: string | null
74  scenarios: ScenarioView[]
75  // Verdicts of scenarios no runner is working on; the ones being worked are in running and stalled.
76  counts: Record<Verdict, number>
77  running: number
78  stalled: number
79  steps: { done: number; total: number }
80  // Share of laned steps a model settled: the run's cost. helper, code and literal steps took no request.
81  // Steps recorded without a lane are left out, so a runner that skips --lane flatters this.
82  llmPct: number | null
83  findings: number
84  flaky: number
85  isDone: boolean
86  // Time since anything was written, while some scenario is stalled; 0 otherwise.
87  quietMs: number
88  isVisible: boolean
89}
90
91// Same rule as report.js verdictOf: passed only when every expected step has a record and all of them
92// passed. A skipped record makes it incomplete, as it does in `report summary`.
93export function verdictOf(row: ScenarioRow, phase: string | null): Verdict {
94  if (row.recorded === null) return SETTLED.has(phase ?? '') ? 'not_run' : 'queued'
95  if ((row.failed ?? 0) > 0) return 'failed'
96  if (row.recorded >= row.expected && row.passed === row.recorded) return 'passed'
97  return 'incomplete'
98}
99
100// An attempt stalls only if it began in the current phase. One left unfinished by an earlier phase is just
101// incomplete: the orchestrator moved on (repair analysis, flaky review), and no runner owes it a record.
102// Working from the active attempt rather than the verdict one is what catches a hung focus replay.
103export function activityOf(row: ScenarioRow, phaseAt: number | null, now: number): Activity {
104  if (row.active_recorded === null || row.active_ts === null) return null
105  if ((row.active_failed ?? 0) > 0 || row.active_recorded >= row.expected) return null
106  if (phaseAt !== null && (row.active_started ?? 0) < phaseAt) return null
107  return now - row.active_ts < STALL_MS ? 'running' : 'stalled'
108}
109
110export function view(snap: Snapshot, now: number): RunView | null {
111  const data: RunData | null = snap.data
112  if (!data) return null
113
114  const counts: Record<Verdict, number> = { passed: 0, failed: 0, incomplete: 0, queued: 0, not_run: 0 }
115  let running = 0
116  let stalled = 0
117  let done = 0
118  let total = 0
119  let latest: ScenarioRow | null = null
120  const scenarios = data.scenarios.map(row => {
121    const verdict = verdictOf(row, data.phase)
122    const activity = activityOf(row, data.phase_at, now)
123    if (activity === 'running') running += 1
124    else if (activity === 'stalled') stalled += 1
125    // An attempt still being worked isn't incomplete yet; it shows as running or stalled instead.
126    if (!(activity && verdict === 'incomplete')) counts[verdict] += 1
127    total += row.expected
128    done += Math.min(row.recorded ?? 0, row.expected)
129    if (row.active_ts !== null && (latest === null || row.active_ts > (latest.active_ts ?? 0))) latest = row
130    return { row, verdict, activity }
131  })
132
133  const lanes = Object.entries(data.lanes).filter(([lane]) => lane !== 'unknown')
134  const settled = lanes.reduce((sum, [, n]) => sum + n, 0)
135  const model = data.lanes.model ?? 0
136  const lastWrite = Math.max(data.phase_at ?? 0, data.initialized_at ?? 0, data.last_ts ?? 0)
137  const isDone = data.phase === 'done'
138  const isVisible = isDone ? now - (data.phase_at ?? 0) < DONE_MS : now - lastWrite < IDLE_MS
139
140  return {
141    run: snap.run,
142    phase: data.phase ?? 'setup',
143    pass: (latest as ScenarioRow | null)?.active_run_id ?? null,
144    scenarios,
145    counts,
146    running,
147    stalled,
148    steps: { done, total },
149    llmPct: settled ? Math.round((100 * model) / settled) : null,
150    findings: Object.values(data.findings).reduce((sum, n) => sum + n, 0),
151    flaky: data.flaky,
152    isDone,
153    quietMs: stalled > 0 ? now - lastWrite : 0,
154    isVisible,
155  }
156}
157
158export function ago(ms: number): string {
159  const s = Math.round(ms / 1000)
160  return s < 90 ? `${s}s` : `${Math.round(s / 60)}m`
161}
162
163// Quiet while most steps run without a model, loud once the run leans on one.
164export function llmColor(pct: number): string | undefined {
165  return pct > 50 ? 'red' : pct > 25 ? 'yellow' : undefined
166}
167

1// One scenario as run.db has it: the manifest row, the attempt that decides its verdict, and the attempt
2// being worked on now (the same one, except during a focus pass or a re-run). Attempt fields are null when
3// the scenario has no record yet.
4export type ScenarioRow = {
5  feature: string
6  name: string
7  expected: number
8  run_id: string | null
9  recorded: number | null
10  passed: number | null
11  failed: number | null
12  active_run_id: string | null
13  active_recorded: number | null
14  active_failed: number | null
15  // first and last record of the active attempt
16  active_started: number | null
17  active_ts: number | null
18}
19
20export type Failure = { id: number; scenario: string; step: string; reason: string | null }
21
22// What one read of run.db returns.
23export type RunData = {
24  phase: string | null
25  phase_at: number | null
26  initialized_at: number | null
27  last_ts: number | null
28  scenarios: ScenarioRow[]
29  lanes: Record<string, number>
30  findings: Record<string, number>
31  flaky: number
32  failures: Failure[]
33}
34
35export type Snapshot = {
36  // `run-7`, and the run dir relative to the project root
37  run: string
38  dir: string
39  data: RunData | null
40  error: string | null
41  checkedAt: number
42}
43
44export type Seen = { dir: string; failId: number }
45
46declare module 'claude-code' {
47  interface PluginState {
48    'testing': { snapshot: Snapshot | null; seen: Seen | null }
49  }
50}
51