signal-lance-qa

1// Signal Lance QA panel: the watch pane (claude/signal-lance-qa-harness.md, step 2b). Tails qa-runs/<batch>/live.jsonl
2// and shows one row per tester; pick a row to follow that tester's actions, thoughts, findings and latest screenshot.
3// Pause / Stop write qa-runs/<batch>/PAUSE and STOP (non-empty = on), which the lead checks before each spawn.
4import { atom, read, update } from 'claude-code'
5import type { Register } from 'claude-code'
6
7import { emptyWatch, fold, header, parseTail, rowText, sevText } from './logic.ts'
8
9const watch = atom({ plugin: 'signal-lance-qa', key: 'watch' } as const, null)
10const follow = atom({ plugin: 'signal-lance-qa', key: 'follow' } as const, '')
11const paused = atom({ plugin: 'signal-lance-qa', key: 'paused' } as const, false)
12
13const PANE = 'sl-qa-watch'
14let ticking: any = null
15
16async function findRoot($: any): Promise<string | null> {
17  const cwd: string = String(await $.session.cwd()).replace(/\\/g, '/')
18  for (const dir of [cwd, cwd.replace(/\/[^/]+\/?$/, ''), cwd.replace(/\/[^/]+\/[^/]+\/?$/, '')]) {
19    if (await $.fs.exists(`${dir}/mods/signal-lance-qa/tool/qa.mjs`)) return dir
20  }
21  return null
22}
23// The batch to watch: the one named, else qa-runs/CURRENT (the lead writes it), else the newest by name
24async function pickBatch($: any, root: string, asked: string): Promise<string> {
25  if (asked) return asked
26  const cur = (await $.fs.read(`${root}/qa-runs/CURRENT`).catch(() => '')).trim()
27  if (cur) return cur
28  const dirs = ((await $.fs.list(`${root}/qa-runs`).catch(() => [])) as any[]).filter(d => d.kind === 'dir').map(d => d.name as string).sort()
29  return dirs.at(-1) || ''
30}
31
32async function poll($: any) {
33  const root = await findRoot($); if (!root) return
34  const w = await read($, watch); if (!w || !w.batch) return
35  const text = await $.fs.read(`${root}/qa-runs/${w.batch}/live.jsonl`).catch(() => '')
36  const { events, offset } = parseTail(text, w.offset)
37  if (events.length || offset !== w.offset) await update($, watch, cur => (cur && cur.batch === w.batch ? fold(offset < cur.offset ? emptyWatch(cur.batch) : cur, events, offset) : cur))
38}
39
40async function openWatch($: any, asked = '') {
41  const root = await findRoot($)
42  if (!root) return 'Not in the Signal Lance repo (no mods/signal-lance-qa/tool/qa.mjs here).'
43  const batch = await pickBatch($, root, asked.trim())
44  if (!batch) return 'No QA batch yet (qa-runs/ is empty). Start one with the sl-qa skill.'
45  const cur = await read($, watch)
46  if (!cur || cur.batch !== batch) { await update($, watch, () => emptyWatch(batch)); await update($, follow, () => '') }
47  await update($, paused, () => false)
48  await poll($)
49  if (!ticking) ticking = $.clock.every(1500, () => { void poll($) })
50  await $.ui.open({ id: PANE, title: 'QA panel · ' + batch })
51  return 'Watching QA batch ' + batch + '.'
52}
53
54export const register: Register = on => {
55  on('session.start', async ($, e, next) => {
56    await $.command.register({ name: 'sl-qa-watch', description: 'Watch a QA batch: one row per tester, follow one live (optional: batch name)' })
57    return next(e)
58  })
59
60  on('command.run', { command: 'sl-qa-watch' }, async ($, e) => ({ text: await openWatch($, e.args ?? '') }))
61
62  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
63    const { Box, Text, Button } = $.ui.resolve(e)
64    const ui: any = $.ui.resolve(e)
65    const w = await read($, watch), f = await read($, follow), isPaused = await read($, paused)
66    if (!w) return <Box><Text dimColor>No batch. /sl-qa-watch [batch]</Text></Box>
67    const root = (await findRoot($)) ?? ''
68    const flag = async (name: string, isOn: boolean) => {
69      await $.fs.write(`${root}/qa-runs/${w.batch}/${name}`, isOn ? new Date().toISOString() : '') // empty = off (no delete in $.fs)
70    }
71    const rows = [...w.rows].sort((a, b) => (a.state === b.state ? a.id.localeCompare(b.id) : a.state === 'running' ? -1 : 1))
72    const lines = e.viewport?.rows ?? 30
73    const sel = f ? w.rows.find(r => r.id === f) : null
74    const feed = sel ? w.feed[sel.id] || [] : []
75    const room = Math.max(4, lines - (sel ? 22 : 8) - (sel ? 0 : rows.length))
76    const shotFile = sel && sel.shot ? `${root}/qa-runs/${sel.shot}` : ''
77    const color = (icon: string) => (icon === '⚑' ? 'warning' : icon === '⚠' ? 'error' : icon === '…' ? 'suggestion' : icon === '↯' ? 'warning' : undefined)
78    return (
79      <Box flexDirection="column">
80        <Text bold wrap="truncate-end">{header(w)}</Text>
81        <Box flexDirection="row">
82          <Button key="pause" label={isPaused ? 'Resume' : 'Pause'} onPress={async () => { const p = !(await read($, paused)); await update($, paused, () => p); await flag('PAUSE', p) }} />
83          <Text> </Text>
84          <Button key="stop" label="Stop batch" onPress={async () => { await flag('STOP', true); await update($, paused, () => true) }} />
85          {sel ? <Text> </Text> : null}
86          {sel ? <Button key="all" label="All testers" onPress={async () => { await update($, follow, () => '') }} /> : null}
87        </Box>
88        {isPaused ? <Text color="warning">Paused: no new testers start; running ones finish their session.</Text> : null}
89        <Text> </Text>
90        {!sel && rows.length === 0 ? <Text dimColor>No testers yet.</Text> : null}
91        {!sel ? rows.map(r => (
92          <Box key={'r' + r.id} flexDirection="column">
93            <Box flexDirection="row">
94              <Button key={'b' + r.id} label="follow" plain onPress={async () => { await update($, follow, () => r.id) }} />
95              <Text color={r.state === 'running' ? 'suggestion' : 'inactive'} wrap="truncate-end"> {rowText(r)}</Text>
96            </Box>
97            {r.thought ? <Text dimColor wrap="truncate-end">   … {r.thought}</Text> : null}
98            {r.last ? <Text dimColor wrap="truncate-end">   ▸ {r.last}</Text> : null}
99          </Box>
100        )) : null}
101        {sel ? (
102          <Box flexDirection="column">
103            <Text bold wrap="truncate-end">{rowText(sel)}</Text>
104            <Text dimColor>findings {sevText(sel.sev)} · oracle {sel.oracle} · {sel.state}</Text>
105            {shotFile && e.surface === 'terminal' && ui.Image
106              ? <ui.Image key="shot" source={{ file: shotFile, format: 'png' }} columns={Math.min(96, Math.max(30, (e.props?.bodyColumns ?? 80) - 2))} rows={14} alt={'latest screenshot ' + sel.shot} />
107              : shotFile ? <Text dimColor wrap="truncate-end">latest screenshot: {shotFile}</Text> : null}
108            <Text> </Text>
109            {feed.slice(-room).map((l, k) => <Text key={'f' + k} color={color(l.icon)} wrap="truncate-end">{l.icon} {l.text}</Text>)}
110          </Box>
111        ) : null}
112      </Box>
113    )
114  })
115}
116

1// The watch pane's pure side: fold qa-runs/<batch>/live.jsonl events into one row per tester session.
2import type { Feed, Row, Watch } from '../types'
3
4export const SEV = ['blocker', 'major', 'minor', 'polish'] as const
5
6export const emptyWatch = (batch: string): Watch => ({ batch, rows: [], feed: {}, planned: 0, oracle: 0, findings: 0, sev: {}, offset: 0 })
7
8const short = (s: string, n: number) => (s.length > n ? s.slice(0, n - 1) + '…' : s)
9
10// One feed line per event, for the follow view
11export function feedLine(e: any): Feed | null {
12  switch (e.kind) {
13    case 'start': return { t: e.t, icon: '●', text: 'start · ' + e.meta.persona + ' · ' + e.meta.model + ' · ' + e.meta.device + ' · seed ' + e.meta.seed + (e.meta.hand ? ' · hand ' + e.meta.hand : '') }
14    case 'action': return { t: e.t, icon: '▸', text: e.cmd + ' ' + (e.target ?? '') + (e.result ? ' → ' + e.result : '') }
15    case 'look': return { t: e.t, icon: '◉', text: 'look · ' + (e.screens || []).join('+'), shot: e.shot }
16    case 'think': return { t: e.t, icon: '…', text: e.text }
17    case 'note': return { t: e.t, icon: '⚑', text: e.severity + ' ' + e.category + ': ' + e.title }
18    case 'fallback': return { t: e.t, icon: '↯', text: 'fallback ' + e.action + (e.why ? ' (' + e.why + ')' : '') }
19    case 'oracle': return { t: e.t, icon: '⚠', text: 'oracle: ' + e.violations.join(' · ') }
20    case 'checkpoint': return { t: e.t, icon: '↻', text: 'checkpoint ' + e.k }
21    case 'end': return { t: e.t, icon: '■', text: 'end · ' + e.notes + ' notes · ' + e.summary }
22  }
23  return null
24}
25
26// Fold new events into the watch (pure: returns a new value). FEED_MAX lines kept per session.
27export const FEED_MAX = 200
28export function fold(w: Watch, events: any[], offset: number): Watch {
29  const rows = new Map(w.rows.map(r => [r.id, { ...r, sev: { ...r.sev } }]))
30  const feed: Record<string, Feed[]> = { ...w.feed }
31  let { oracle, findings } = w
32  const sev = { ...w.sev }
33  for (const e of events) {
34    if (e.kind === 'plan') { w = { ...w, planned: e.sessions }; continue }
35    if (!e.session) continue
36    let r = rows.get(e.session)
37    if (!r) { r = { id: e.session, persona: '', model: '', device: '', state: 'running', screen: '', last: '', thought: '', notes: 0, sev: {}, oracle: 0, actions: 0, shot: '', t: e.t }; rows.set(e.session, r) }
38    r.t = e.t
39    if (e.kind === 'start') Object.assign(r, { persona: e.meta.persona, model: e.meta.model, device: e.meta.device, state: 'running' })
40    if (e.kind === 'action' || e.kind === 'fallback') { r.actions++; r.last = short(feedLine(e)!.text, 80) }
41    if (e.kind === 'look') { r.screen = (e.screens || []).join('+'); if (e.shot) r.shot = e.shot }
42    const moved = e.kind === 'action' ? /screen \S+ → (\S+)/.exec(e.result || '') : null
43    if (moved) r.screen = moved[1] ?? r.screen
44    if (e.kind === 'think') r.thought = short(e.text, 100)
45    if (e.kind === 'note') { r.notes++; r.sev[e.severity] = (r.sev[e.severity] || 0) + 1; findings++; sev[e.severity] = (sev[e.severity] || 0) + 1 }
46    if (e.kind === 'oracle') { r.oracle += e.violations.length; oracle += e.violations.length }
47    if (e.kind === 'end') r.state = 'done'
48    const l = feedLine(e)
49    if (l) feed[e.session] = [...(feed[e.session] || []), l].slice(-FEED_MAX)
50  }
51  return { ...w, rows: [...rows.values()], feed, oracle, findings, sev, offset }
52}
53
54// Parse the new tail of live.jsonl (from a byte/char offset); a half-written last line waits for the next read
55export function parseTail(text: string, offset: number): { events: any[]; offset: number } {
56  if (text.length < offset) offset = 0 // the file was replaced: start over
57  const chunk = text.slice(offset), end = chunk.lastIndexOf('\n')
58  if (end < 0) return { events: [], offset }
59  const events: any[] = []
60  for (const line of chunk.slice(0, end).split('\n')) { if (!line.trim()) continue; try { events.push(JSON.parse(line)) } catch { /* skip a bad line */ } }
61  return { events, offset: offset + end + 1 }
62}
63
64export const sevText = (s: Record<string, number>) => SEV.filter(k => s[k]).map(k => k.charAt(0).toUpperCase() + s[k]).join(' ') || '—'
65export function header(w: Watch) {
66  const run = w.rows.filter(r => r.state === 'running').length, done = w.rows.filter(r => r.state === 'done').length
67  const queued = Math.max(0, w.planned - w.rows.length)
68  return 'batch ' + w.batch + ' · ' + run + ' running · ' + done + ' done' + (w.planned ? ' · ' + queued + ' queued' : '') + ' · ' + w.findings + ' findings (' + sevText(w.sev) + ') · oracle ' + w.oracle
69}
70export function rowText(r: Row) {
71  return (r.state === 'running' ? '●' : '■') + ' ' + r.id + ' · ' + r.persona + ' · ' + r.model + ' · ' + r.device + ' · ' + (r.screen || '?') + ' · ' + r.actions + ' acts · ' + sevText(r.sev) + (r.oracle ? ' · ⚠' + r.oracle : '')
72}
73

1export type Feed = { t: string; icon: string; text: string; shot?: string }
2export type Row = {
3  id: string; persona: string; model: string; device: string; state: 'running' | 'done'
4  screen: string; last: string; thought: string; notes: number; sev: Record<string, number>; oracle: number
5  actions: number; shot: string; t: string
6}
7export type Watch = {
8  batch: string; rows: Row[]; feed: Record<string, Feed[]>; planned: number
9  oracle: number; findings: number; sev: Record<string, number>; offset: number
10}
11
12declare module 'claude-code' {
13  interface PluginState {
14    'signal-lance-qa': { watch: Watch | null; follow: string; paused: boolean }
15  }
16}
17