playwright-claude-mod

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { TestItem, TestStatus } from '../types'
5
6const PANE = 'playwright'
7const TITLE = 'Playwright'
8const dir = atom({ plugin: 'playwright-claude-mod', key: 'dir' } as const, '')
9const tests = atom({ plugin: 'playwright-claude-mod', key: 'tests' } as const, [])
10const isBusy = atom({ plugin: 'playwright-claude-mod', key: 'isBusy' } as const, false)
11const message = atom({ plugin: 'playwright-claude-mod', key: 'message' } as const, '')
12const needsReporter = atom({ plugin: 'playwright-claude-mod', key: 'needsReporter' } as const, false)
13
14const CONFIGS = ['ts', 'js', 'mts', 'mjs', 'cts', 'cjs'].map(ext => `playwright.config.${ext}`)
15// Matches `['json', { outputFile: 'results.json' }]` in the config's reporter list.
16const JSON_REPORTER = /\[\s*['"]json['"]\s*,\s*\{[^}]*outputFile\s*:\s*['"`]([^'"`]+)['"`]/
17// The json reporter's file, set for one run: `PLAYWRIGHT_JSON_OUTPUT_NAME=out.json npx playwright test`.
18const JSON_OUTPUT_ENV = /\bPLAYWRIGHT_JSON_OUTPUT_(?:NAME|FILE)=(['"]?)([^'"\s]+)\1/
19const HINT = "No json reporter with an outputFile. Add ['json', { outputFile: 'test-results/results.json' }] to reporter in the Playwright config."
20const FIX_PROMPT = (cwd: string) =>
21  `Add a json reporter to the Playwright config in ${cwd}: ['json', { outputFile: 'test-results/results.json' }]. Keep the existing reporters.`
22// A command that starts `playwright test`: at the line start, after ; & | ( or after env assignments,
23// optionally through npx, pnpm (exec), yarn or bunx. A mention inside quotes does not match.
24const PLAYWRIGHT_TEST = /(?:^|[;&|(]\s*)(?:\w+=\S*\s+)*(?:npx\s+|pnpm\s+(?:exec\s+)?|yarn\s+|bunx\s+)?playwright\s+test\b/m
25const NO_REPORT = "This run wrote no json report, so the results did not change. A --reporter flag replaces the config's reporters."
26// A --reporter flag drops the config's reporters, json included, so the agent is told how to keep it.
27const REPORTER_NOTE = {
28  id: 'playwright-claude-mod:json-reporter',
29  scope: 'session',
30  text: "A side panel shows Playwright results from the json reporter's outputFile in the Playwright config. When you pass --reporter to `playwright test`, also include json and set PLAYWRIGHT_JSON_OUTPUT_NAME to that outputFile, for example `PLAYWRIGHT_JSON_OUTPUT_NAME=test-results/results.json npx playwright test --reporter=list,json`. Without --reporter, the config's reporters already write it.",
31} as const
32
33const ICONS: Record<TestStatus, string> = {
34  idle: '○',
35  running: '◌',
36  passed: '✓',
37  failed: '✗',
38  skipped: '–',
39  flaky: '!',
40}
41const COLORS: Record<TestStatus, string | undefined> = {
42  idle: 'inactive',
43  running: 'warning',
44  passed: 'success',
45  failed: 'error',
46  skipped: 'inactive',
47  flaky: 'warning',
48}
49
50type Spec = {
51  id: string
52  title: string
53  file: string
54  line: number
55  tests: { status?: string; results: { status: string; duration: number; error?: { message?: string } }[] }[]
56}
57type Suite = { title: string; file: string; line: number; specs: Spec[]; suites?: Suite[] }
58
59// Playwright prints colored errors; the pane draws plain text.
60const stripAnsi = (text: string) => text.replace(/\u001b\[[0-9;]*m/g, '')
61
62function statusOf(spec: Spec): TestStatus {
63  const test = spec.tests[0]
64  if (!test || test.results.length === 0) return 'idle'
65  if (test.status === 'flaky') return 'flaky'
66  if (test.status === 'skipped') return 'skipped'
67  return test.status === 'expected' ? 'passed' : 'failed'
68}
69
70function flatten(suites: Suite[], parents: string[] = []): TestItem[] {
71  return suites.flatMap(suite => {
72    // The top suite is the file; its title adds nothing to a test's name.
73    const path = suite.line === 0 ? parents : [...parents, suite.title]
74    const own = suite.specs.map(spec => {
75      const last = spec.tests[0]?.results.at(-1)
76      const error = last?.error?.message
77      return {
78        id: spec.id,
79        file: spec.file,
80        line: spec.line,
81        title: [...path, spec.title].join(' › '),
82        status: statusOf(spec),
83        durationMs: last?.duration,
84        error: error ? stripAnsi(error).split('\n').slice(0, 4).join('\n') : undefined,
85      }
86    })
87    return [...own, ...flatten(suite.suites ?? [], path)]
88  })
89}
90
91function parseReport(text: string): TestItem[] {
92  const start = text.indexOf('{')
93  if (start < 0) throw new Error('no JSON report')
94  const report = JSON.parse(text.slice(start)) as { suites: Suite[]; errors?: { message: string }[] }
95  const fatal = report.errors?.[0]?.message
96  if (fatal && report.suites.length === 0) throw new Error(stripAnsi(fatal).split('\n')[0])
97  return flatten(report.suites)
98}
99
100// 0 when the file does not exist yet.
101const modifiedAt = ($: EngineInterface, path: string) => $.fs.stat(path).then(stat => stat.mtimeMs, () => 0)
102
103const projectDir = async ($: EngineInterface) => (await read($, dir)) || (await $.session.cwd())
104
105// The json reporter's outputFile from the project's config, as an absolute path.
106async function findReportFile($: EngineInterface, cwd: string): Promise<string | undefined> {
107  for (const name of CONFIGS) {
108    if (!(await $.fs.exists(`${cwd}/${name}`))) continue
109    const outputFile = JSON_REPORTER.exec(await $.fs.read(`${cwd}/${name}`))?.[1]
110    return outputFile && absolute(cwd, outputFile)
111  }
112  return undefined
113}
114
115const absolute = (cwd: string, file: string) => (file.startsWith('/') ? file : `${cwd}/${file.replace(/^\.\//, '')}`)
116
117// With a json reporter configured, runs with the config's own reporters and reads its file;
118// without one, asks for JSON on stdout.
119async function playwright($: EngineInterface, args: string[]): Promise<TestItem[]> {
120  const cwd = await projectDir($)
121  const reportFile = args.includes('--list') ? undefined : await findReportFile($, cwd)
122  const reporter = reportFile ? [] : ['--reporter=json']
123  const before = reportFile && (await modifiedAt($, reportFile))
124  const run = await $.process.run(['npx', 'playwright', 'test', ...args, ...reporter], {
125    cwd,
126    // The html reporter serves its report after a failure and waits; that would hold the panel busy.
127    env: { PW_TEST_HTML_REPORT_OPEN: 'never' },
128    timeoutMs: 600_000,
129  })
130  try {
131    if (reportFile && (await modifiedAt($, reportFile)) === before) throw new Error(NO_REPORT)
132    return parseReport(reportFile ? await $.fs.read(reportFile) : run.stdout)
133  } catch (error) {
134    throw new Error(stripAnsi(run.stderr).trim() || (error as Error).message)
135  }
136}
137
138// Results of a partial run update the tests they cover and keep the rest.
139function merge(list: TestItem[], ran: TestItem[]): TestItem[] {
140  const results = new Map(ran.map(test => [test.id, test]))
141  const known = new Set(list.map(test => test.id))
142  return [...list.map(test => results.get(test.id) ?? test), ...ran.filter(test => !known.has(test.id))]
143}
144
145const summary = (ran: TestItem[]) => {
146  const failed = ran.filter(test => test.status === 'failed').length
147  return failed ? `${failed} of ${ran.length} failed.` : `${ran.length} passed.`
148}
149
150// Checked and set with no await between, so two quick presses cannot both start a run.
151let isRunning = false
152
153async function busy($: EngineInterface, label: string, work: () => Promise<string>) {
154  if (isRunning) return
155  isRunning = true
156  try {
157    await update($, isBusy, () => true)
158    await update($, message, () => label)
159    const done = await work()
160    await update($, message, () => done)
161  } catch (error) {
162    await update($, message, () => `Error: ${(error as Error).message}`)
163  } finally {
164    isRunning = false
165    await update($, isBusy, () => false)
166  }
167}
168
169const discover = ($: EngineInterface) =>
170  busy($, 'Loading tests…', async () => {
171    const found = await playwright($, ['--list'])
172    const known = new Map((await read($, tests)).map(test => [test.id, test]))
173    // Keep the last result of each test we already ran.
174    await update($, tests, () => found.map(test => ({ ...test, ...pick(known.get(test.id)) })))
175    const hasReport = await findReportFile($, await projectDir($))
176    await update($, needsReporter, () => !hasReport)
177    return hasReport ? `Found ${found.length} tests.` : `Found ${found.length} tests. ${HINT}`
178  })
179
180const pick = (test: TestItem | undefined) =>
181  test && { status: test.status, durationMs: test.durationMs, error: test.error }
182
183// The config can take many shapes, so Claude makes the edit and the person reviews it.
184async function fixReporter($: EngineInterface) {
185  await update($, needsReporter, () => false)
186  await update($, message, () => 'Asked Claude to add the json reporter.')
187  await $.prompt.submit({ text: FIX_PROMPT(await projectDir($)), asUser: true })
188}
189
190const runTests = ($: EngineInterface, only?: TestItem) =>
191  busy($, only ? `Running ${only.title}…` : 'Running all tests…', async () => {
192    await update($, tests, list =>
193      list.map(test => (!only || test.id === only.id ? { ...test, status: 'running' as const } : test)),
194    )
195    try {
196      const ran = await playwright($, only ? [`${only.file}:${only.line}`] : [])
197      await update($, tests, list => (only ? merge(list, ran) : ran))
198      return summary(ran)
199    } finally {
200      // A failed run, or a report that does not hold the test, leaves nothing marked running.
201      await update($, tests, list =>
202        list.map(test => (test.status === 'running' ? { ...test, status: 'idle' as const } : test)),
203      )
204    }
205  })
206
207export const register: Register = on => {
208  on('session.start', async ($, e, next) => {
209    await $.command.register({
210      name: 'playwright',
211      description: 'Show Playwright tests and results in a side panel',
212      argumentHint: '[project folder]',
213    })
214    return next(e)
215  })
216
217  on('command.run', { command: 'playwright' }, async ($, e) => {
218    void $.ui.status(undefined)
219    const folder = e.args.trim() || (await $.session.cwd())
220    await update($, dir, () => folder)
221    await $.ui.open({ id: PANE, title: TITLE })
222    void discover($)
223    return { text: `Playwright panel opened for ${folder}.` }
224  })
225
226  // When Claude runs the tests through Bash, show what the json reporter wrote.
227  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
228    // A background run returns before Playwright writes its report, so there is nothing to read yet.
229    if (!PLAYWRIGHT_TEST.test(e.command) || /--list\b/.test(e.command) || e.run_in_background) return next(e)
230    const cwd = await projectDir($)
231    // A file the command names for its run wins over the config's.
232    const named = JSON_OUTPUT_ENV.exec(e.command)?.[2]
233    const reportFile = named ? absolute(cwd, named) : await findReportFile($, cwd)
234    const before = reportFile && (await modifiedAt($, reportFile))
235    const ran = await next(e)
236    if (ran.deny !== undefined) return ran
237    try {
238      await update($, needsReporter, () => !reportFile)
239      if (!reportFile) {
240        await update($, message, () => HINT)
241      } else if ((await modifiedAt($, reportFile)) === before) {
242        await update($, message, () => NO_REPORT)
243      } else {
244        const results = parseReport(await $.fs.read(reportFile))
245        await update($, tests, list => merge(list, results))
246        await update($, message, () => `Claude's run: ${summary(results)}`)
247      }
248    } catch (error) {
249      await update($, message, () => `Error: ${(error as Error).message}`)
250    }
251    // An unasked pane waits on a narrow terminal, or opens as a tab behind another pane (the diff
252    // panel), and only `focus` raises it, which takes the keyboard. Anything short of a pane known
253    // to be shown, a failed open included, gets the status line.
254    const isShown = await $.ui
255      .open({ id: PANE, title: TITLE })
256      .then(async opened => opened.isPlaced && (await $.ui.panes()).some(pane => pane.id === PANE && pane.isShown))
257      .catch(() => false)
258    void $.ui.status(isShown ? undefined : `Playwright: ${await read($, message)} Open the Playwright tab or type /playwright.`)
259    return ran
260  }).catch(($, e, next) => next(e))
261
262  on('prompt.compose', async ($, e, next) => {
263    const composed = await next(e)
264    return { sections: [...composed.sections, REPORTER_NOTE] }
265  })
266
267  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
268    const { Box, Text, Button } = $.ui.resolve(e)
269    const list = await read($, tests)
270    const folder = await read($, dir)
271    const working = await read($, isBusy)
272    const note = await read($, message)
273    const canFix = await read($, needsReporter)
274    const count = (status: TestStatus) => list.filter(test => test.status === status).length
275    const files = [...new Set(list.map(test => test.file))]
276
277    return (
278      <Box flexDirection="column">
279        <Text dimColor wrap="truncate-start">{folder}</Text>
280        <Box flexDirection="row" gap={1}>
281          <Button key="run-all" hotkey="r" variant="primary" onPress={() => runTests($)}>
282            Run all
283          </Button>
284          <Button key="refresh" hotkey="l" onPress={() => discover($)}>
285            Refresh
286          </Button>
287          {canFix && (
288            <Button key="fix-reporter" hotkey="f" onPress={() => fixReporter($)}>
289              Fix
290            </Button>
291          )}
292        </Box>
293        <Text>
294          <Text color="success">✓ {count('passed')}</Text>{'  '}
295          <Text color="error">✗ {count('failed')}</Text>{'  '}
296          <Text dimColor>○ {count('idle')}  total {list.length}</Text>
297        </Text>
298        {note !== '' && <Text color={working ? 'warning' : undefined} wrap="wrap">{note}</Text>}
299        {files.map(file => (
300          <Box flexDirection="column" marginTop={1}>
301            <Text bold wrap="truncate-start">{file}</Text>
302            {list
303              .filter(test => test.file === file)
304              .map(test => (
305                <Box flexDirection="column">
306                  <Box flexDirection="row" gap={1}>
307                    <Text color={COLORS[test.status]}>{ICONS[test.status]}</Text>
308                    <Button key={test.id} plain dimColor={test.status === 'idle'} onPress={() => runTests($, test)}>
309                      {test.title}
310                    </Button>
311                    {test.durationMs !== undefined && test.status !== 'running' && (
312                      <Text dimColor>{test.durationMs}ms</Text>
313                    )}
314                  </Box>
315                  {test.status === 'failed' && test.error && (
316                    <Box paddingLeft={2}>
317                      <Text color="error" wrap="wrap">{test.error}</Text>
318                    </Box>
319                  )}
320                </Box>
321              ))}
322          </Box>
323        ))}
324      </Box>
325    )
326  })
327}
328

1export type TestStatus = 'idle' | 'running' | 'passed' | 'failed' | 'skipped' | 'flaky'
2
3export type TestItem = {
4  id: string
5  file: string
6  line: number
7  title: string
8  status: TestStatus
9  durationMs?: number
10  error?: string
11}
12
13declare module 'claude-code' {
14  interface PluginState {
15    'playwright-claude-mod': {
16      dir: string
17      tests: TestItem[]
18      isBusy: boolean
19      needsReporter: boolean
20      message: string
21    }
22  }
23}
24