agent-canvas

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4// A live board for the session in a Claude Code pane. bridge/bridge.mjs runs headless Chrome on
5// the board page (or any URL) and writes frames as PNG files; an Image shows them, swapped in with
6// $.ui.blit; a Client laid over it (input.tsx) catches clicks, keys and the pane size. While the
7// board is on, the rules in RULES.md join the system prompt, and the Stop event is blocked until
8// the agent has written this turn's turn.html.
9const PANE = 'agent-canvas'
10const NO_IMAGES = 'This terminal draws no images for Claude Code. Use Ghostty, kitty or WezTerm.'
11const NO_IMAGES_HERDR = 'Images are off: this claude started without CLAUDE_CODE_FORCE_TERMINAL_IMAGES=1. Set it in your shell for Herdr, then start claude again.'
12const isOnRef = atom({ plugin: 'agent-canvas', key: 'isOn' } as const, false)
13
14let dir = '' // this session's board files
15let rules = ''
16let home = ''
17let maxFps = 60
18let scale = '2'
19let noImages = NO_IMAGES
20let url = '' // what the pane shows: '' is the board
21
22let bridge: AsyncGenerator<unknown, unknown> | null = null
23let socket: string | null = null
24let frame: { file: string; n: number } | null = null // the newest frame the bridge wrote
25let shownN = 0
26let isBlitting = false
27let note = 'starting Chrome'
28let stats = { frames: 0, ms: 0, since: Date.now(), text: '' }
29let seen = { instance: '', seq: 0 }
30let size: { t: 'resize'; columns: number; rows: number } | null = null // the pane's last size, sent again once Chrome is up
31let chain: Promise<unknown> = Promise.resolve()
32
33let turnStart = 0 // when the current prompt arrived
34let lastWork = 0 // when the last tool call that was not a board write finished
35let isLoopTick = false // a /loop firing: a quiet tick may leave the board alone
36
37const toUrl = (s: string) => (/^[a-z]+:\/\//.test(s) ? s : `file://${s.replace(/^~/, home)}`)
38
39// Input goes to the bridge in order, one request at a time.
40function send($: EngineInterface, ev: object) {
41  const at = socket
42  if (!at) return chain
43  chain = chain.then(() =>
44    $.http.fetch('http://bridge/', { method: 'POST', body: JSON.stringify(ev), socketPath: at }).catch(() => undefined))
45  return chain
46}
47
48// One blit at a time, always of the newest frame: frames that arrive meanwhile are skipped, so a
49// slow terminal shows fewer frames instead of falling further behind.
50async function show($: EngineInterface) {
51  if (isBlitting) return
52  isBlitting = true
53  while (frame && frame.n !== shownN) {
54    const { file, n } = frame
55    const t = Date.now()
56    const r = await $.ui.blit({ requestId: PANE, key: 'view', source: { file, format: 'png', generation: n } })
57    if (r.deny) {
58      if (r.deny.includes('alt')) note = noImages
59      $.ui.invalidate('ui.render') // not mounted yet, resized, or no images: draw the tree again
60      break
61    }
62    shownN = n
63    stats.frames++
64    stats.ms += Date.now() - t
65    if (Date.now() - stats.since >= 1000) { // the footer's frame rate, redrawn once a second
66      stats = { frames: 0, ms: 0, since: Date.now(), text: `${stats.frames} fps` }
67      $.ui.invalidate('ui.render')
68    }
69    if (maxFps < 60) await $.clock.sleep(Math.max(0, 1000 / maxFps - (Date.now() - t)))
70  }
71  isBlitting = false
72}
73
74function start($: EngineInterface) {
75  if (bridge) return
76  const argv = ['node', `${$.plugin.root}/bridge/bridge.mjs`, '--board', dir, ...(url ? [url] : [])]
77  const child = $.process.spawn({ argv, env: { AGENT_CANVAS_SCALE: scale, AGENT_CANVAS_FPS: String(maxFps) } })
78  bridge = child
79  void (async () => {
80    let buf = ''
81    try {
82      for await (const chunk of child) {
83        if (chunk.stream === 'stderr') { note = chunk.text.trim().slice(0, 160); continue }
84        buf += chunk.text
85        for (let i; (i = buf.indexOf('\n')) >= 0;) {
86          const m = JSON.parse(buf.slice(0, i)); buf = buf.slice(i + 1)
87          if (m.socket) { socket = m.socket; note = ''; if (size) void send($, size); $.ui.invalidate('ui.render') }
88          if (m.frame) frame = { file: m.frame, n: m.n }
89          if (m.error) note = m.error
90        }
91        void show($) // not awaited: the loop keeps reading, show() picks up the newest frame
92      }
93    } catch (err) {
94      note = `bridge failed: ${String(err).slice(0, 160)}`
95    }
96    bridge = null
97    socket = null
98    frame = null
99    $.ui.invalidate('ui.render')
100  })()
101}
102
103function stop() {
104  void bridge?.return(undefined) // leaving the stream ends the child, and the bridge closes Chrome
105  bridge = null
106}
107
108async function turnOn($: EngineInterface) {
109  await update($, isOnRef, () => true)
110  await $.ui.open({ id: PANE, title: 'Canvas' })
111  start($)
112}
113
114export const register: Register = on => {
115  on('session.start', async ($, e, next) => {
116    await $.command.register({ name: 'agent-canvas', description: 'Live board for this session (off: close it; a URL or path: show that page)' })
117    const started = await next(e)
118    home = (await $.env.get('HOME')) ?? ''
119    dir = `${home}/.claude/agent-canvas/sessions/${await $.session.id()}`
120    rules = (await $.fs.read(`${$.plugin.root}/RULES.md`)).replaceAll('{{DIR}}', dir)
121    maxFps = Number(await $.env.get('AGENT_CANVAS_FPS')) || 60
122    scale = (await $.env.get('AGENT_CANVAS_SCALE')) || '2'
123    if (await $.env.get('HERDR_ENV')) noImages = NO_IMAGES_HERDR
124    if ((await $.env.get('AGENT_CANVAS_OPEN')) === '1' || (await read($, isOnRef))) await turnOn($)
125    return started
126  })
127
128  on('command.run', { command: 'agent-canvas' }, async ($, e) => {
129    const arg = e.args.trim()
130    if (arg === 'off') {
131      await update($, isOnRef, () => false)
132      stop()
133      await $.ui.close({ id: PANE })
134      return { text: 'Canvas off.' }
135    }
136    url = arg && arg !== 'board' ? toUrl(arg) : ''
137    await send($, { t: 'navigate', url }) // a running bridge switches page; '' is the board
138    await turnOn($)
139    return { text: url ? `Canvas shows ${url}` : `Canvas on: ${dir}` }
140  })
141
142  // The rules ride in the system prompt while the board is on, so a compaction cannot lose them.
143  on('prompt.compose', async ($, e, next) => {
144    const composed = await next(e)
145    if (!(await read($, isOnRef))) return composed
146    return { sections: [...composed.sections, { id: 'agent-canvas:rules', text: rules, scope: 'session' as const }] }
147  })
148
149  on('prompt.submit', async ($, e, next) => {
150    turnStart = Date.now()
151    lastWork = 0
152    isLoopTick = /^\s*\/loop\b|<<autonomous-loop/.test(e.text)
153    if (await read($, isOnRef)) await $.fs.write(`${dir}/turn`, String(turnStart)) // the board marks panels written after this
154    return next(e)
155  })
156
157  on('tool.call', async ($, e, next) => {
158    const result = await next(e)
159    if (!JSON.stringify(e).includes(dir)) lastWork = Date.now() // board writes are not work the board must describe
160    return result
161  })
162
163  on('classic.Stop', async ($, e, next) => {
164    const result = await next(e)
165    if (!(await read($, isOnRef)) || e.stop_hook_active || isLoopTick || !turnStart) return result
166    const file = `${dir}/turn.html`
167    const written = (await $.fs.exists(file)) ? (await $.fs.stat(file)).mtimeMs : 0
168    if (written < turnStart) {
169      return { ...result, block: `The canvas is on but ${file} was not written this turn. Write this reply's turn.html, update the panels that changed and append to log.txt, as the canvas rules say, then finish.` }
170    }
171    if (written < lastWork) {
172      return { ...result, block: `${file} was written before the rest of this turn's work, so it describes the plan instead of the result. Rewrite it to say what is true now, then finish.` }
173    }
174    return result
175  })
176
177  on('ui.message', async ($, e, next) => {
178    if (e.requestId !== PANE) return next(e)
179    const data = e.data as { instance: string; events: { seq: number; t: string }[] }
180    if (data.instance !== seen.instance) seen = { instance: data.instance, seq: 0 }
181    for (const ev of data.events) {
182      if (ev.seq <= seen.seq) continue
183      seen.seq = ev.seq
184      if (ev.t === 'resize') size = ev as unknown as typeof size
185      void send($, ev)
186    }
187    await chain
188    return {}
189  })
190
191  on('ui.scroll', async ($, e, next) => {
192    if (e.requestId !== PANE) return next(e)
193    await send($, { t: 'wheel', dy: e.by })
194    return { deny: 'the page scrolls itself' }
195  })
196
197  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
198    const ui = $.ui.resolve(e)
199    if (e.surface !== 'terminal' || !('Image' in ui)) {
200      const { Text } = ui
201      return <Text>The canvas needs a terminal that draws images (Ghostty, kitty, WezTerm).</Text>
202    }
203    const { Box, Text, Image, Client } = ui
204    const columns = Math.max(20, e.props.bodyColumns)
205    const rows = Math.max(5, e.props.scroll.bodyRows - 1)
206    const where = url ? url.replace(/^file:\/\//, '') : 'board'
207    return (
208      <Box flexDirection="column">
209        <Box width={columns} height={rows}>
210          {frame
211            ? <Image key="view" source={{ file: frame.file, format: 'png', generation: frame.n }} columns={columns} rows={rows} alt={noImages} />
212            : <Text dimColor>{note || 'starting Chrome'}</Text>}
213          <Box position="absolute" top={0} left={0}>
214            <Client key="input" module="./input.tsx" props={{ columns, rows }} width={columns} height={rows} />
215          </Box>
216        </Box>
217        <Text dimColor wrap="truncate-end">{[where, note, stats.text, 'click to use the page, Esc gives the keys back'].filter(Boolean).join(' · ')}</Text>
218      </Box>
219    )
220  })
221}
222

1import type { ClientModule } from 'claude-code'
2
3type Props = { columns: number; rows: number }
4type State = { columns: number; rows: number }
5type Ev = { t: string; seq: number; [k: string]: unknown }
6
7// Laid over the browser frame. Sends the pointer, keys and its own size to the hooks module.
8// A post replaces one not yet delivered in the same frame, so each post carries the last
9// 20 events with a sequence number; the hooks module skips the ones it has seen.
10const instance = Math.random().toString(36).slice(2)
11let seq = 0
12let pending: Ev[] = []
13
14function emit(post: (d: never) => void, ev: Omit<Ev, 'seq'>) {
15  const next = { ...ev, seq: ++seq } as Ev
16  const last = pending[pending.length - 1]
17  if (next.t === 'move' && last?.t === 'move') pending[pending.length - 1] = next // keep only the newest hover
18  else pending.push(next)
19  pending = pending.slice(-20)
20  post({ instance, events: pending } as never)
21}
22
23const Input: ClientModule<Props, State> = (props, surface) => {
24  const post = surface.post as (d: never) => void
25  if (surface.state === undefined) {
26    surface.onPointer(ev => {
27      if (ev.type === 'enter' || ev.type === 'leave') return
28      emit(post, { t: ev.type, x: ev.fine?.x ?? ev.x + 0.5, y: ev.fine?.y ?? ev.y + 0.5, button: ev.button ?? null })
29    })
30    surface.onKey(k => emit(post, { t: 'key', key: k.key, ctrl: !!k.ctrl, shift: !!k.shift, meta: !!k.meta }))
31  }
32  const s = surface.state
33  if (surface.columns > 0 && (s === undefined || s.columns !== surface.columns || s.rows !== surface.rows)) {
34    emit(post, { t: 'resize', columns: surface.columns, rows: surface.rows })
35    surface.setState({ columns: surface.columns, rows: surface.rows })
36  } else if (s === undefined) {
37    surface.setState({ columns: 0, rows: 0 })
38  }
39  const { Box } = surface.elements
40  return <Box width={props.columns} height={props.rows} />
41}
42
43export default Input
44

1/** Whether the board is on for this session: kept across a plugin reload. */
2export type IsOn = boolean
3
4declare module 'claude-code' {
5  interface PluginState {
6    'agent-canvas': { isOn: IsOn }
7  }
8}
9