screenshot-pane

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { Shot } from '../types'
5
6const PANE = 'screenshots'
7const IMAGE_EXT = /\.(png|jpe?g|webp|gif)$/i
8const PNG_EXT = /\.png$/i
9const ABSOLUTE_IMAGE = /(\/[^\s"'`]+\.(?:png|jpe?g))/g
10const NO_IMAGES = "can't draw images here (needs Ghostty, kitty or WezTerm, outside tmux) · o opens it"
11// agent-browser flags that take a value, so the parser skips it
12const FLAGS_WITH_VALUE = new Set([
13  '--session',
14  '--screenshot-dir',
15  '--screenshot-quality',
16  '--screenshot-format',
17])
18const NAVIGATE = new Set(['open', 'goto', 'navigate'])
19
20const shotsRef = { plugin: 'screenshot-pane', key: 'shots' } as const
21const indexRef = { plugin: 'screenshot-pane', key: 'index' } as const
22const dismissedRef = { plugin: 'screenshot-pane', key: 'dismissed' } as const
23const compareRef = { plugin: 'screenshot-pane', key: 'compare' } as const
24const shots = atom(shotsRef, [] as Shot[])
25const index = atom(indexRef, -1)
26const dismissed = atom(dismissedRef, false)
27const compare = atom(compareRef, false)
28
29export type Found = { path: string; source: Shot['source'] }
30export type Parsed = { shots: Found[]; page?: string }
31
32/** Splits one shell segment into words, honouring simple quotes. */
33function words(segment: string): string[] {
34  const out: string[] = []
35  const re = /"([^"]*)"|'([^']*)'|(\S+)/g
36  let m: RegExpExecArray | null
37  while ((m = re.exec(segment)) !== null) out.push(m[1] ?? m[2] ?? m[3] ?? '')
38  return out
39}
40
41function normalize(path: string): string {
42  const parts: string[] = []
43  for (const p of path.split('/')) {
44    if (p === '' || p === '.') continue
45    if (p === '..') parts.pop()
46    else parts.push(p)
47  }
48  return '/' + parts.join('/')
49}
50
51function resolvePath(dir: string, home: string, p: string): string {
52  if (p.startsWith('/')) return normalize(p)
53  if (p === '~' || p.startsWith('~/')) return normalize(home + p.slice(1))
54  return normalize(dir + '/' + p)
55}
56
57/** The positional words after an agent-browser subcommand, flags and their values skipped. */
58function agentBrowserArgs(t: string[]): { sub: string; args: string[] } | null {
59  const ab = t.indexOf('agent-browser')
60  if (ab < 0) return null
61  let i = ab + 1
62  while (i < t.length && (t[i] ?? '').startsWith('-')) i += FLAGS_WITH_VALUE.has(t[i] ?? '') ? 2 : 1
63  const sub = t[i]
64  if (sub === undefined) return null
65  const args: string[] = []
66  for (let j = i + 1; j < t.length; j += 1) {
67    const tok = t[j] ?? ''
68    if (tok.startsWith('-')) {
69      if (FLAGS_WITH_VALUE.has(tok)) j += 1
70      continue
71    }
72    args.push(tok)
73  }
74  return { sub, args }
75}
76
77/**
78 * What a Bash command does that this mod cares about: the image files it writes
79 * (`agent-browser screenshot [sel] [path]`, `screencapture ... path`), resolved
80 * against the cwd its own `cd`s lead to, and the last page it opens in the
81 * browser. A path holding `$` cannot be resolved here and is skipped.
82 */
83export function parseCommand(command: string, cwd: string, home: string): Parsed {
84  const found: Found[] = []
85  let page: string | undefined
86  let dir = cwd
87  for (const raw of command.split(/&&|\|\||;|\n|\|/)) {
88    const t = words(raw.trim())
89    if (t.length === 0) continue
90    if (t[0] === 'cd' && t[1] !== undefined && !t[1].includes('$')) {
91      dir = resolvePath(dir, home, t[1])
92      continue
93    }
94    const ab = agentBrowserArgs(t)
95    if (ab !== null) {
96      if (NAVIGATE.has(ab.sub) && ab.args[0] !== undefined) page = ab.args[0]
97      if (ab.sub !== 'screenshot') continue
98      const last = ab.args[ab.args.length - 1]
99      if (last !== undefined && (IMAGE_EXT.test(last) || last.includes('/'))) {
100        found.push({ path: resolvePath(dir, home, last), source: 'agent-browser' })
101      }
102      continue
103    }
104    if (t[0] === 'screencapture') {
105      const last = t[t.length - 1]
106      if (last !== undefined && IMAGE_EXT.test(last)) {
107        found.push({ path: resolvePath(dir, home, last), source: 'screencapture' })
108      }
109    }
110  }
111  return { shots: found.filter(f => !f.path.includes('$')), page }
112}
113
114/** A page address short enough for a header: host and path for the web, the file name for file://. */
115export function pageLabel(url: string): string {
116  const file = url.match(/^file:\/\/(.*)$/)
117  if (file) return (file[1] ?? '').split('/').pop() ?? url
118  const web = url.match(/^https?:\/\/([^?#]*)/)
119  const label = web ? (web[1] ?? url).replace(/\/$/, '') : url
120  return label.length <= 40 ? label : label.slice(0, 39) + '…'
121}
122
123const B64 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'
124
125function decodeBase64(chars: string): number[] {
126  const out: number[] = []
127  let bits = 0
128  let have = 0
129  for (const c of chars) {
130    const v = B64.indexOf(c)
131    if (v < 0) break
132    bits = (bits << 6) | v
133    have += 6
134    if (have >= 8) {
135      have -= 8
136      out.push((bits >> have) & 0xff)
137    }
138  }
139  return out
140}
141
142/** Width and height from a PNG's IHDR chunk, read off the file's base64. */
143export function pngSize(base64: string): { width: number; height: number } | null {
144  if (!base64.startsWith('iVBOR')) return null
145  // base64 chars 20..31 hold bytes 15..23; width is bytes 16..19, height 20..23
146  const b = decodeBase64(base64.slice(20, 32))
147  if (b.length < 9) return null
148  const at = (i: number): number => b[i] ?? 0
149  const width = ((at(1) << 24) | (at(2) << 16) | (at(3) << 8) | at(4)) >>> 0
150  const height = ((at(5) << 24) | (at(6) << 16) | (at(7) << 8) | at(8)) >>> 0
151  return width > 0 && height > 0 ? { width, height } : null
152}
153
154/**
155 * Cells for a picture of w×h pixels across `columns` cells, a cell `ratio`
156 * times as tall as wide. A tall picture keeps its width and the pane scrolls;
157 * only the 255-row limit of one Image narrows it.
158 */
159export function fitWidth(w: number, h: number, columns: number, ratio: number) {
160  let cols = Math.max(1, Math.min(255, Math.floor(columns)))
161  let rows = Math.max(1, Math.round((cols * h) / w / ratio))
162  if (rows > 255) {
163    rows = 255
164    cols = Math.max(1, Math.round((rows * ratio * w) / h))
165  }
166  return { columns: cols, rows }
167}
168
169/** The shot shown "before" this one: the latest earlier shot of the same page, else of the same file. */
170export function previousOf(list: readonly Shot[], i: number): number {
171  const cur = list[i]
172  if (cur === undefined) return -1
173  for (let j = i - 1; j >= 0; j -= 1) {
174    const s = list[j]
175    if (s === undefined) continue
176    if (cur.page !== undefined ? s.page === cur.page : s.origin === cur.origin) return j
177  }
178  return -1
179}
180
181export function ago(ms: number): string {
182  const s = Math.max(0, Math.round(ms / 1000))
183  if (s < 60) return 'just now'
184  if (s < 3600) return `${Math.floor(s / 60)}m ago`
185  return `${Math.floor(s / 3600)}h ago`
186}
187
188function shortPath(path: string, home: string): string {
189  const p = home && path.startsWith(home) ? '~' + path.slice(home.length) : path
190  return p.length <= 40 ? p : '…/' + p.split('/').slice(-2).join('/')
191}
192
193async function home($: EngineInterface): Promise<string> {
194  return (await $.env.get('HOME')) ?? ''
195}
196
197async function cacheDir($: EngineInterface): Promise<string> {
198  return `${await home($)}/.cache/screenshot-pane/${await $.session.id()}`
199}
200
201async function openFile($: EngineInterface, path: string): Promise<void> {
202  const mac = await $.process.run(['open', path]).catch(() => ({ exitCode: 1 }))
203  if (mac.exitCode !== 0) await $.process.run(['xdg-open', path]).catch(() => undefined)
204}
205
206type Options = { autoOpen: boolean; history: number; cellRatio: number }
207
208/**
209 * Records an image file the session just produced or looked at: copies it
210 * into this session's cache so a later write to the same path can't change
211 * it, then shows the pane, or toasts when the person closed it.
212 */
213async function addShot(
214  $: EngineInterface,
215  opts: Options,
216  origin: string,
217  source: Shot['source'],
218  page: string | undefined,
219  agent: string | undefined,
220): Promise<void> {
221  let stat
222  try {
223    stat = await $.fs.stat(origin)
224  } catch {
225    return
226  }
227  if (stat.kind !== 'file') return
228  const generation = Math.floor(stat.mtimeMs)
229  const { value: known = [] } = await $.state.get(shotsRef)
230  if (known.some(s => s.origin === origin && s.at >= generation)) return
231
232  const isPng = PNG_EXT.test(origin)
233  let size: { width: number; height: number } | null = null
234  if (isPng) {
235    try {
236      size = pngSize((await $.fs.read(origin, { as: 'bytes' })).base64)
237    } catch {
238      size = null
239    }
240  }
241  const n = (known[known.length - 1]?.n ?? 0) + 1
242  const dir = await cacheDir($)
243  const copy = `${dir}/${n}-${origin.split('/').pop() ?? 'shot.png'}`
244  const copied = await $.process
245    .run(['mkdir', '-p', dir])
246    .then(() => $.process.run(['cp', origin, copy]))
247    .catch(() => ({ exitCode: 1 }))
248
249  const shot: Shot = {
250    n,
251    path: copied.exitCode === 0 ? copy : origin,
252    origin,
253    at: Math.max(generation, await $.clock.now()),
254    width: size?.width ?? 1600,
255    height: size?.height ?? 1000,
256    isPng,
257    source,
258    ...(page !== undefined ? { page } : {}),
259    ...(agent !== undefined ? { agent } : {}),
260  }
261  await update($, shots, list => [...list, shot].slice(-Math.max(1, opts.history)))
262  await update($, compare, () => false)
263
264  const { value: isDismissed = false } = await $.state.get(dismissedRef)
265  const pane = (await $.ui.panes()).find(p => p.id === PANE)
266  if (pane === undefined && opts.autoOpen && !isDismissed) {
267    void $.ui.open({ id: PANE, title: 'Screenshots' })
268  } else if (pane?.isShown !== true) {
269    $.ui.toast(`screenshot ${n}${page ? ` of ${pageLabel(page)}` : ''} · /screenshots to view`)
270  }
271}
272
273export const register: Register = (on, options) => {
274  const opts: Options = {
275    autoOpen: options.autoOpen !== false,
276    history: typeof options.history === 'number' ? options.history : 50,
277    cellRatio: typeof options.cellRatio === 'number' && options.cellRatio > 0 ? options.cellRatio : 2.4,
278  }
279  // The page each agent's browser was last sent to; a reload forgets it.
280  const pages = new Map<string, string>()
281
282  on('session.start', async ($, e, next) => {
283    await $.command.register({
284      name: 'screenshots',
285      description: 'Show the screenshots Claude took (`clear` empties the list)',
286    })
287    // Sessions older than a week leave their cached copies behind; sweep them.
288    void $.process
289      .run(['find', `${await home($)}/.cache/screenshot-pane`, '-mindepth', '1', '-maxdepth', '1', '-mtime', '+7', '-exec', 'rm', '-rf', '{}', '+'])
290      .catch(() => undefined)
291    return next(e)
292  })
293
294  on('command.run', { command: 'screenshots' }, async ($, e) => {
295    if (e.args.trim() === 'clear') {
296      await update($, shots, () => [])
297      await update($, index, () => -1)
298      await update($, compare, () => false)
299      return { text: 'Screenshot list cleared.' }
300    }
301    await update($, dismissed, () => false)
302    await $.ui.open({ id: PANE, title: 'Screenshots', focus: true })
303    const { value: list = [] } = await $.state.get(shotsRef)
304    return { text: `Screenshots pane opened (${list.length} so far).` }
305  })
306
307  on('ui.close', async ($, e, next) => {
308    if (e.id === PANE && e.origin.kind === 'person') await update($, dismissed, () => true)
309    return next(e)
310  })
311
312  on('tool.call', { tool: 'Bash' }, async ($, e, next) => {
313    if (!/agent-browser|\bscreencapture\b/.test(e.command)) return next(e)
314    const agent = e.agentId ?? ''
315    const parsed = parseCommand(e.command, await $.session.cwd(), await home($))
316    const ran = await next(e)
317    if (ran.deny !== undefined || ran.isError === true) return ran
318    if (parsed.page !== undefined) pages.set(agent, parsed.page)
319    const paths = parsed.shots.map(s => s.path)
320    if (paths.length === 0 && /\bscreenshot\b/.test(e.command) && ran.text !== undefined) {
321      // `agent-browser screenshot` with no path prints where it saved the file
322      for (const m of ran.text.matchAll(ABSOLUTE_IMAGE)) if (m[1] !== undefined) paths.push(m[1])
323    }
324    const source = parsed.shots[0]?.source ?? 'agent-browser'
325    const page = source === 'agent-browser' ? pages.get(agent) : undefined
326    for (const p of paths) await addShot($, opts, p, source, page, e.agentId)
327    return ran
328  }).catch(($, e, next) => next(e))
329
330  on('tool.call', { tool: 'Read' }, async ($, e, next) => {
331    if (!IMAGE_EXT.test(e.file_path)) return next(e)
332    const ran = await next(e)
333    if (ran.deny === undefined && ran.isError !== true) {
334      const path = resolvePath(await $.session.cwd(), await home($), e.file_path)
335      await addShot($, opts, path, 'read', undefined, e.agentId)
336    }
337    return ran
338  }).catch(($, e, next) => next(e))
339
340  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
341    const { Box, Text, Button } = $.ui.resolve(e)
342    const list = await read($, shots)
343    const pick = await read($, index)
344    const isComparing = await read($, compare)
345    const cur = pick < 0 ? list.length - 1 : Math.min(pick, list.length - 1)
346    const before = isComparing ? previousOf(list, cur) : -1
347    const shown = before >= 0 ? before : cur
348    const shot = list[shown]
349
350    if (shot === undefined) {
351      return (
352        <Box flexDirection="column">
353          <Text dimColor>No screenshots yet.</Text>
354          <Text dimColor>Screenshots Claude takes with agent-browser or screencapture, and images it reads, show up here.</Text>
355        </Box>
356      )
357    }
358
359    const now = await $.clock.now()
360    const where = shot.page !== undefined ? pageLabel(shot.page) : shortPath(shot.origin, await home($))
361    const who = shot.agent !== undefined ? ` · subagent ${shot.agent.slice(0, 6)}` : ''
362    const hasBefore = previousOf(list, cur) >= 0
363    const { columns, rows } = fitWidth(shot.width, shot.height, e.props.bodyColumns - 1, opts.cellRatio)
364
365    // Handlers read the live values, never the ones captured while drawing.
366    const position = async (): Promise<{ i: number; n: number }> => {
367      const { value: all = [] } = await $.state.get(shotsRef)
368      const { value: i = -1 } = await $.state.get(indexRef)
369      return { i: i < 0 ? all.length - 1 : Math.min(i, all.length - 1), n: all.length }
370    }
371    // The shot on screen now: the "before" one while comparing.
372    const onScreen = async (): Promise<Shot | undefined> => {
373      const { value: all = [] } = await $.state.get(shotsRef)
374      const { value: isBefore = false } = await $.state.get(compareRef)
375      const { i } = await position()
376      const j = isBefore ? previousOf(all, i) : -1
377      return all[j >= 0 ? j : i]
378    }
379
380    const picture =
381      e.surface === 'terminal' && shot.isPng ? (
382        (() => {
383          const { Image } = $.ui.resolve(e)
384          return (
385            <Image
386              key="shot"
387              source={{ file: shot.path, format: 'png', generation: shot.n }}
388              columns={columns}
389              rows={rows}
390              alt={NO_IMAGES}
391            />
392          )
393        })()
394      ) : (
395        <Text dimColor>{shot.isPng ? NO_IMAGES : `${shot.origin} · only PNG is drawn here · o opens it`}</Text>
396      )
397
398    return (
399      <Box flexDirection="column">
400        <Box>
401          <Text bold>{`${cur + 1}/${list.length}`}</Text>
402          {before >= 0 && <Text color="yellow">{` before (#${shot.n})`}</Text>}
403          <Text>{`  ${where}`}</Text>
404          <Text dimColor>{`  ${ago(now - shot.at)}${who}`}</Text>
405        </Box>
406        <Box>
407          <Button
408            plain
409            dimColor
410            key="prev"
411            label="prev"
412            hotkey="h"
413            onPress={async () => {
414              const { i } = await position()
415              await update($, compare, () => false)
416              await update($, index, () => Math.max(0, i - 1))
417            }}
418          />
419          <Text dimColor>{'  '}</Text>
420          <Button
421            plain
422            dimColor
423            key="next"
424            label="next"
425            hotkey="l"
426            onPress={async () => {
427              const { i, n } = await position()
428              await update($, compare, () => false)
429              await update($, index, () => (i + 1 >= n - 1 ? -1 : i + 1))
430            }}
431          />
432          <Text dimColor>{'  '}</Text>
433          {hasBefore && (
434            <Button
435              plain
436              dimColor
437              key="before"
438              label={isComparing ? 'after' : 'before'}
439              hotkey="b"
440              onPress={() => update($, compare, c => !c)}
441            />
442          )}
443          {hasBefore && <Text dimColor>{'  '}</Text>}
444          <Button
445            plain
446            dimColor
447            key="comment"
448            label="comment"
449            hotkey="c"
450            onPress={async () => {
451              const s = await onScreen()
452              if (s === undefined) return
453              const label = s.page !== undefined ? ` of ${pageLabel(s.page)}` : ''
454              const filled = await $.prompt.fill({ text: `[screenshot #${s.n}${label}: ${s.path}] `, mode: 'insert' })
455              $.ui.toast(filled.isFilled ? 'Reference added to your prompt · Esc to type' : 'Could not reach the prompt box')
456            }}
457          />
458          <Text dimColor>{'  '}</Text>
459          <Button
460            plain
461            dimColor
462            key="open"
463            label="open"
464            hotkey="o"
465            onPress={async () => {
466              const s = await onScreen()
467              if (s !== undefined) await openFile($, s.path)
468            }}
469          />
470        </Box>
471        {picture}
472      </Box>
473    )
474  })
475}
476

1export type Shot = {
2  /** 1-based number, stable for the session. */
3  n: number
4  /** The copy this mod keeps, so a later screenshot to the same path can't change it. */
5  path: string
6  /** Where Claude wrote it. */
7  origin: string
8  /** When it was noticed, milliseconds since the epoch. */
9  at: number
10  /** Pixel size; a 16:10 guess when the header could not be read. */
11  width: number
12  height: number
13  /** True for a PNG the terminal can draw. */
14  isPng: boolean
15  /** What produced it. */
16  source: 'agent-browser' | 'screencapture' | 'read'
17  /** The page the browser was on, when known. */
18  page?: string
19  /** The subagent that took it; absent for the main conversation. */
20  agent?: string
21}
22
23declare module 'claude-code' {
24  interface PluginState {
25    'screenshot-pane': {
26      /** Every screenshot noticed this session, oldest first, capped. */
27      shots: Shot[]
28      /** Which one the pane shows: an index, or -1 to follow the newest. */
29      index: number
30      /** True once the person closed the pane; new shots then toast instead. */
31      dismissed: boolean
32      /** True while the pane shows the previous shot of the same page. */
33      compare: boolean
34    }
35  }
36}
37