screen-context

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, PromptOrigin, Register } from 'claude-code'
3
4import type { PendingCapture } from '../types'
5
6/** What the Helper writes for each Capture (helper/Sources/ScreenContext/Capture.swift). */
7type Capture = {
8  id: string
9  takenAt: number
10  app: string
11  window: string
12  /** Absent when the Helper may not record the screen. */
13  screenshot?: string
14  /** A small JPEG of the Screenshot, as a data URL. */
15  thumbnail?: string
16  text: string
17  /** The CLI session id of the session the Capture went to. */
18  session: string
19}
20
21/** Where a message the person typed comes from: the terminal, Remote Control, the desktop app (an SDK host). */
22const fromPerson: PromptOrigin['kind'][] = ['composer', 'bridge', 'sdk']
23
24/** The Pending captures this session's bar lists. */
25const bar = atom({ plugin: 'screen-context', key: 'bar' } as const, [] as PendingCapture[])
26
27export const register: Register = on => {
28  on('session.start', async ($, e, next) => {
29    const started = await next(e)
30    let shown = ''
31    // The Helper drops Captures for this session into the folder; list them in the bar.
32    $.clock.every(1_000, async () => {
33      // Asked every time: after a /clear the process goes on under a new session id.
34      const me = await $.session.id()
35      const mine = (await pendingCaptures($)).filter(capture => capture.session === me)
36      const ids = mine.map(capture => capture.id).join()
37      if (ids === shown) return
38      shown = ids
39      await update($, bar, () => mine.map(({ id, app, window, thumbnail }) => ({ id, app, window, thumbnail })))
40    })
41    return started
42  })
43
44  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
45    const captures = await read($, bar)
46    if (captures.length === 0) return next(e)
47    const { Box, Button, Text } = $.ui.resolve(e)
48    // The terminal has no Svg: it names each capture instead.
49    const Svg = e.surface === 'terminal' ? undefined : $.ui.resolve(e).Svg
50    return (
51      <Box flexDirection="row" flexWrap="wrap" gap={1}>
52        {captures.map(capture => {
53          const name = capture.window ? `${capture.app} — ${capture.window}` : capture.app
54          return (
55            // Keyed, so hovering the thumbnail brings its × in.
56            <Box key={capture.id}>
57              {capture.thumbnail && Svg ? (
58                <Svg source={thumbnailSvg(capture.thumbnail)} alt={name} />
59              ) : (
60                <Text wrap="truncate-end">{name}</Text>
61              )}
62              {/* Parked above the band, which clips it, and moved onto the top-right corner on hover:
63                  the desktop draws a display="none" reveal as a card above the thumbnail instead. */}
64              <Box position="absolute" top={-50} right={0} hover={{ top: 0 }}>
65                <Button key={`remove-${capture.id}`} label="×" variant="primary" onPress={() => settle($, capture.id, 'removed from the bar')} />
66              </Box>
67            </Box>
68          )
69        })}
70      </Box>
71    )
72  })
73
74  on('prompt.submit', async ($, e, next) => {
75    if (!fromPerson.includes(e.origin.kind)) return next(e)
76    const me = await $.session.id()
77    const going = (await pendingCaptures($)).filter(capture => capture.session === me)
78    if (going.length === 0) return next(e)
79    const result = await next({ ...e, context: [...(e.context ?? []), ...going.map(describe)] })
80    // Settled only once the message went out: a hook beneath can still stop it.
81    if (!result.drop) for (const capture of going) await settle($, capture.id, `sent in session ${me}`)
82    return result
83  })
84}
85
86async function folder($: EngineInterface) {
87  return `${await $.env.get('HOME')}/.claude/screen-context`
88}
89
90/** Every Pending capture, whichever session it waits for, oldest first. */
91async function pendingCaptures($: EngineInterface): Promise<Capture[]> {
92  const dir = await folder($)
93  const names = new Set((await $.fs.list(dir).catch(() => [])).map(entry => entry.name))
94  const captures: Capture[] = []
95  for (const name of names) {
96    // `<id>.done` marks a Capture already sent or removed.
97    if (!name.endsWith('.json') || names.has(name.replace(/json$/, 'done'))) continue
98    try {
99      captures.push(JSON.parse(String(await $.fs.read(`${dir}/${name}`))))
100    } catch {}
101  }
102  return captures.sort((a, b) => a.takenAt - b.takenAt)
103}
104
105/** Marks a Capture as dealt with, so neither the bar nor a later message picks it up again. */
106async function settle($: EngineInterface, id: string, how: string) {
107  await $.fs.write(`${await folder($)}/${id}.done`, how).catch(() => undefined)
108  await update($, bar, captures => captures.filter(capture => capture.id !== id))
109}
110
111/**
112 * The Screenshot's thumbnail inside an SVG (the desktop draws no raster image for a mod), with 8 px of room above
113 * and right of it, so the × in the SVG's corner sits across the thumbnail's.
114 */
115function thumbnailSvg(dataUrl: string): string {
116  return `<svg xmlns="http://www.w3.org/2000/svg" width="136" height="88" viewBox="0 0 136 88"><clipPath id="c"><rect y="8" width="128" height="80" rx="6"/></clipPath><image href="${dataUrl}" y="8" width="128" height="80" preserveAspectRatio="xMidYMid slice" clip-path="url(#c)"/></svg>`
117}
118
119function describe(capture: Capture): string {
120  const where = capture.window ? `${capture.app}, window "${capture.window}"` : capture.app
121  return [
122    `The user captured their screen while in ${where}.`,
123    capture.screenshot
124      ? `Screenshot of the whole display: ${capture.screenshot}. Open it with the Read tool before answering.`
125      : 'No screenshot was taken.',
126    capture.text
127      ? `The window's text, read through macOS accessibility (it can include parts scrolled out of view):\n<screen-text>\n${capture.text}\n</screen-text>`
128      : 'No text could be read from the window.',
129  ].join('\n')
130}
131

1/** A Capture waiting to go out with this session's next message, as its bar lists it. */
2export type PendingCapture = {
3  id: string
4  app: string
5  window: string
6  /** A small JPEG of the Screenshot, as a data URL; absent without one. */
7  thumbnail?: string
8}
9
10declare module 'claude-code' {
11  interface PluginState {
12    'screen-context': { bar: PendingCapture[] }
13  }
14}
15