lightbox

1import type { EngineInterface, PromptBox, Register } from 'claude-code'
2import { atom, read, update } from 'claude-code'
3
4import type { Caption, PastedImage, Shown } from '../types'
5import {
6  type CaptionModel,
7  captionCommand,
8  captionLabel,
9  captionModelOf,
10  cleanCaption,
11} from './captions'
12import { header } from './header'
13import { parseOd, pngSize } from './png'
14import { findTags, imageNumbers, tagAtCursor } from './tags'
15import { tierOf } from './tokens'
16import { expanded, thumbnails } from './view'
17
18// Pasting raises no prompt.edit and a click on a tag only moves the cursor,
19// so the prompt is also read on a timer.
20const POLL_MS = 150
21// Below this the band can't show a picture under its border and header.
22const MIN_ROWS = 4
23// A caption run usually takes about five seconds.
24const CAPTION_TIMEOUT_MS = 60_000
25
26const EMPTY: Shown = { images: [], focused: null }
27const shown = atom({ plugin: 'lightbox', key: 'shown' } as const, EMPTY)
28const captions = atom(
29  { plugin: 'lightbox', key: 'captions' } as const,
30  {} as Record<string, Caption>,
31)
32
33let isActive = false
34let lastWritten = ''
35let pending: PromptBox | undefined
36let isSyncing = false
37
38// The session's image folder, once found. The project folder is named after
39// the directory the session started in, which may not be the current one, so
40// it is found by the session id instead.
41let found: { sessionId: string; path: string } | undefined
42// A cached file never changes, so each is read once, an unreadable one too.
43const readImages = new Map<string, Exclude<ImageRead, { kind: 'missing' }>>()
44
45let captionModel: CaptionModel | null = null
46// Images waiting for a caption run, oldest first; runs go one at a time.
47const captionQueue: { path: string; model: CaptionModel }[] = []
48let isCaptioning = false
49
50/** What reading a cached image found. */
51type ImageRead =
52  | { kind: 'found'; image: PastedImage }
53  // There, but not a PNG this can read.
54  | { kind: 'unreadable' }
55  // Not written yet, so the next poll looks again.
56  | { kind: 'missing' }
57
58export const register: Register = (on, options) => {
59  const showsThumbnails = options.collapsedView !== 'none'
60  captionModel = captionModelOf(options.captionModel)
61
62  on('session.start', async ($, e, next) => {
63    const result = await next(e)
64    // The desktop app previews pasted images itself. The check also keeps a
65    // caption run, itself a headless Claude Code, from captioning anything.
66    isActive = e.isInteractive && e.surface === 'terminal'
67    if (isActive) {
68      // A reload drops the runs in flight; their images are captioned again.
69      await update($, captions, current => withoutPending(current))
70      $.clock.every(POLL_MS, () => poll($))
71    }
72    return result
73  })
74
75  on('prompt.edit', async ($, e, next) => {
76    const box = await next(e)
77    if (isActive) await sync($, box)
78    return box
79  })
80
81  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
82    if (e.surface !== 'terminal' || e.props.hasSurvey) return next(e)
83    const band = { columns: e.props.bodyColumns, rows: e.props.maxRows }
84    if (band.rows < MIN_ROWS) return next(e)
85
86    const { images, focused } = await read($, shown)
87    const byPath = await read($, captions)
88    const ui = $.ui.resolve(e)
89    const index = images.findIndex(image => image.n === focused)
90    const image = images[index]
91    if (image) {
92      const tier = tierOf(await $.session.model())
93      const caption = captionLabel(byPath[image.path])
94      const position = { index, count: images.length }
95      return expanded(ui, image, header(image, caption, position, tier), band)
96    }
97    if (showsThumbnails && images.length > 0) {
98      const labels = new Map<number, string>()
99      for (const { n, path } of images) {
100        const label = captionLabel(byPath[path])
101        if (label) labels.set(n, label)
102      }
103      return thumbnails(ui, images, labels, band)
104    }
105    return next(e)
106  })
107}
108
109async function poll($: EngineInterface) {
110  await sync($, await $.prompt.read())
111}
112
113// Applies the newest box only, one at a time, so a slow read can't land after
114// a newer one. Never throws: it runs inside the prompt's own edits, and a
115// failed read just leaves the band as it was until the next poll.
116async function sync($: EngineInterface, box: PromptBox) {
117  pending = box
118  if (isSyncing) return
119  isSyncing = true
120  try {
121    while (pending) {
122      const latest = pending
123      pending = undefined
124      await apply($, latest)
125    }
126  } catch (error) {
127    $.ui.log(`lightbox: ${String(error)}`, { to: 'debug' })
128  } finally {
129    isSyncing = false
130  }
131}
132
133async function apply($: EngineInterface, box: PromptBox) {
134  const tags = findTags(box.text)
135  const images = await cachedImages($, imageNumbers(tags))
136  const n = tagAtCursor(tags, box.cursor)?.n
137  const next: Shown = {
138    images,
139    focused: images.some(image => image.n === n) ? (n ?? null) : null,
140  }
141  const key = JSON.stringify(next)
142  if (key === lastWritten) return
143  lastWritten = key
144  await update($, shown, () => next)
145  await queueCaptions($, images)
146}
147
148async function queueCaptions($: EngineInterface, images: PastedImage[]) {
149  const model = captionModel
150  if (model === null) return
151  const known = await read($, captions)
152  const paths = images
153    .map(image => image.path)
154    .filter(path => known[path] === undefined)
155  if (paths.length === 0) return
156  await update($, captions, current => {
157    const next = { ...current }
158    for (const path of paths) next[path] = { status: 'pending' }
159    return next
160  })
161  captionQueue.push(...paths.map(path => ({ path, model })))
162  void runCaptions($)
163}
164
165// Runs the queued captions one at a time, in the background: each takes
166// seconds, and the band redraws as each lands.
167async function runCaptions($: EngineInterface) {
168  if (isCaptioning) return
169  isCaptioning = true
170  try {
171    for (let job = captionQueue.shift(); job; job = captionQueue.shift()) {
172      const { path, model } = job
173      const caption = await captionImage($, path, model)
174      await update($, captions, current => ({ ...current, [path]: caption }))
175    }
176  } catch (error) {
177    $.ui.log(`lightbox: ${String(error)}`, { to: 'debug' })
178  } finally {
179    isCaptioning = false
180  }
181}
182
183async function captionImage(
184  $: EngineInterface,
185  path: string,
186  model: CaptionModel,
187): Promise<Caption> {
188  const slash = path.lastIndexOf('/')
189  const command = captionCommand(model, path.slice(slash + 1))
190  const run = await $.process
191    .run(command, { cwd: path.slice(0, slash), timeoutMs: CAPTION_TIMEOUT_MS })
192    .catch((error: unknown) => ({
193      exitCode: -1,
194      stdout: '',
195      stderr: String(error),
196    }))
197  const text = run.exitCode === 0 ? cleanCaption(run.stdout) : null
198  if (text) return { status: 'done', text }
199  $.ui.log(
200    `lightbox: no caption for ${path} (exit ${run.exitCode}): ${run.stderr.trim()}`,
201    { to: 'debug' },
202  )
203  return { status: 'failed' }
204}
205
206function withoutPending(
207  byPath: Record<string, Caption>,
208): Record<string, Caption> {
209  return Object.fromEntries(
210    Object.entries(byPath).filter(
211      ([, caption]) => caption.status !== 'pending',
212    ),
213  )
214}
215
216/**
217 * The cached images numbered `numbers`, in that order, leaving out any not
218 * written yet. Claude Code caches each paste as
219 * `<tmp>/<project>/<session>/images/<n>.png`.
220 */
221async function cachedImages(
222  $: EngineInterface,
223  numbers: readonly number[],
224): Promise<PastedImage[]> {
225  if (numbers.length === 0) return []
226  const dir = await imagesDir($)
227  if (dir === undefined) return []
228  const images: PastedImage[] = []
229  for (const n of numbers) {
230    const path = `${dir}/${n}.png`
231    let read = readImages.get(path)
232    if (read === undefined) {
233      const result = await readImage($, n, path)
234      if (result.kind === 'missing') continue
235      readImages.set(path, result)
236      read = result
237    }
238    if (read.kind === 'found') images.push(read.image)
239  }
240  return images
241}
242
243async function imagesDir($: EngineInterface): Promise<string | undefined> {
244  const sessionId = await $.session.id()
245  if (found?.sessionId === sessionId) return found.path
246  const root = await tmpRoot($)
247  const entries = await $.fs.list(root).catch(() => [])
248  for (const entry of entries) {
249    const path = `${root}/${entry.name}/${sessionId}/images`
250    if (entry.kind === 'dir' && (await $.fs.exists(path))) {
251      found = { sessionId, path }
252      return path
253    }
254  }
255  return undefined
256}
257
258async function tmpRoot($: EngineInterface): Promise<string> {
259  const configured = await $.env.get('CLAUDE_CODE_TMPDIR')
260  if (configured) return configured
261  const { stdout } = await $.process.run(['id', '-u'])
262  return `/tmp/claude-${stdout.trim()}`
263}
264
265async function readImage(
266  $: EngineInterface,
267  n: number,
268  path: string,
269): Promise<ImageRead> {
270  const stat = await $.fs.stat(path).catch(() => undefined)
271  if (stat === undefined) return { kind: 'missing' }
272  if (stat.kind !== 'file') return { kind: 'unreadable' }
273  // The header alone, so a large screenshot isn't copied in to read 24 bytes.
274  const { exitCode, stdout } = await $.process.run([
275    'od',
276    '-An',
277    '-tx1',
278    '-N24',
279    path,
280  ])
281  const size = exitCode === 0 ? pngSize(parseOd(stdout)) : null
282  if (size === null) return { kind: 'unreadable' }
283  return { kind: 'found', image: { n, path, ...size, bytes: stat.size } }
284}
285

1import type { Caption } from '../types'
2
3export type CaptionModel = 'sonnet' | 'haiku'
4
5/** The caption model setting: a model, or null when captions are off. */
6export function captionModelOf(setting: unknown): CaptionModel | null {
7  if (setting === 'off') return null
8  return setting === 'haiku' ? 'haiku' : 'sonnet'
9}
10
11const SYSTEM_PROMPT =
12  'You caption images a person pasted into a coding assistant, so they can ' +
13  'tell several apart at a glance. Name what it shows specifically: the ' +
14  "app, page, error or thing, and its state, like 'OpenAI pricing page, " +
15  "retention table' or 'TypeScript error in register.tsx'. Reply with the " +
16  'caption alone: at most 8 words, sentence case, no final period.'
17
18/**
19 * A headless Claude Code run that reads `file` (relative to the run's
20 * working directory) and prints its caption. It loads Read alone, no MCP
21 * servers, skills or settings hooks, and isn't kept as a session.
22 */
23export function captionCommand(model: CaptionModel, file: string): string[] {
24  return [
25    'claude',
26    '-p',
27    '--model',
28    model,
29    '--effort',
30    'low',
31    '--tools',
32    'Read',
33    '--allowedTools',
34    'Read',
35    '--strict-mcp-config',
36    '--disable-slash-commands',
37    '--no-session-persistence',
38    '--settings',
39    '{"disableAllHooks":true}',
40    '--system-prompt',
41    SYSTEM_PROMPT,
42    `Caption the image in ./${file}.`,
43  ]
44}
45
46const MAX_LENGTH = 80
47
48/** The caption in a run's output: its first line, unquoted; null if none. */
49export function cleanCaption(output: string): string | null {
50  const line = output
51    .split('\n')
52    .map(text => text.trim())
53    .find(Boolean)
54  if (!line) return null
55  const text = line
56    .replace(/^["'`]+|["'`]+$/g, '')
57    .replace(/\.$/, '')
58    .replace(/\s+/g, ' ')
59    .trim()
60  if (!text) return null
61  return text.length > MAX_LENGTH ? `${text.slice(0, MAX_LENGTH - 1)}…` : text
62}
63
64/** What to show for a caption: its text, a placeholder while it's written. */
65export function captionLabel(caption: Caption | undefined): string | undefined {
66  if (caption?.status === 'pending') return 'describing…'
67  if (caption?.status === 'done') return caption.text
68  return undefined
69}
70

1import type { PastedImage } from '../types'
2import { imageTokens, sentSize, type Tier } from './tokens'
3
4/** The two lines above a large preview. */
5export type Header = {
6  /** Which image and what it shows: `Image #4 · Login page with error`. */
7  title: string
8  /**
9   * Where it sits, its size, and what the model will read:
10   * `2 of 3 · 2850×1672 · 1.1 MB · sent as 2520×1478 · ~4.8k tokens`.
11   */
12  details: string
13}
14
15export function header(
16  image: PastedImage,
17  caption: string | undefined,
18  position: { index: number; count: number },
19  tier: Tier,
20): Header {
21  const sent = sentSize(image, tier)
22  const isScaled = sent.width !== image.width || sent.height !== image.height
23  const details = [
24    position.count > 1 && `${position.index + 1} of ${position.count}`,
25    `${image.width}×${image.height}`,
26    fileSize(image.bytes),
27    isScaled && `sent as ${sent.width}×${sent.height}`,
28    `~${tokenCount(imageTokens(sent))} tokens`,
29  ]
30  return {
31    title: [`Image #${image.n}`, caption].filter(Boolean).join(' · '),
32    details: details.filter(Boolean).join(' · '),
33  }
34}
35
36function fileSize(bytes: number): string {
37  if (bytes < 1024) return `${bytes} B`
38  if (bytes < 1024 * 1024) return `${Math.round(bytes / 1024)} KB`
39  return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
40}
41
42function tokenCount(tokens: number): string {
43  return tokens < 1000 ? String(tokens) : `${(tokens / 1000).toFixed(1)}k`
44}
45

1export type Size = { width: number; height: number }
2
3const SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]
4
5/**
6 * A PNG's pixel size from its first 24 bytes: the signature, then the IHDR
7 * chunk, whose width and height are big-endian at offsets 16 and 20. Null
8 * when the bytes aren't a PNG.
9 */
10export function pngSize(head: Uint8Array): Size | null {
11  if (head.length < 24 || SIGNATURE.some((byte, i) => head[i] !== byte)) {
12    return null
13  }
14  const view = new DataView(head.buffer, head.byteOffset, head.byteLength)
15  const width = view.getUint32(16)
16  const height = view.getUint32(20)
17  return width > 0 && height > 0 ? { width, height } : null
18}
19
20/** Bytes from `od -An -tx1` output: hex pairs separated by whitespace. */
21export function parseOd(output: string): Uint8Array {
22  const pairs = output.trim().split(/\s+/).filter(Boolean)
23  return Uint8Array.from(pairs, pair => Number.parseInt(pair, 16))
24}
25

1/**
2 * One `[Image #n]` tag in the prompt draft, as UTF-16 offsets like the
3 * prompt's cursor.
4 */
5export type ImageTag = { n: number; start: number; end: number }
6
7const TAG = /\[Image #(\d+)\]/g
8
9export function findTags(draft: string): ImageTag[] {
10  return Array.from(draft.matchAll(TAG), match => ({
11    n: Number(match[1]),
12    start: match.index,
13    end: match.index + match[0].length,
14  }))
15}
16
17/**
18 * The tag the cursor touches. The prompt treats a tag as one unit: the
19 * cursor stops only at its edges, a click puts it at the end, and a paste
20 * leaves it there. So a cursor at either edge is on the tag, and where two
21 * tags meet, the one ending there wins.
22 */
23export function tagAtCursor(
24  tags: readonly ImageTag[],
25  cursor: number,
26): ImageTag | undefined {
27  return (
28    tags.find(tag => tag.end === cursor) ??
29    tags.find(tag => tag.start <= cursor && cursor < tag.end)
30  )
31}
32
33/** Each image number the tags name, once, in order of first appearance. */
34export function imageNumbers(tags: readonly ImageTag[]): number[] {
35  return [...new Set(tags.map(tag => tag.n))]
36}
37

1import type { Size } from './png'
2
3/**
4 * A model's image limits. Claude 4.7 and later read images at high
5 * resolution; earlier models at standard. From
6 * https://platform.claude.com/docs/en/build-with-claude/vision
7 */
8export type Tier = { maxEdge: number; maxTokens: number }
9
10export const HIGH_RES: Tier = { maxEdge: 2576, maxTokens: 4784 }
11export const STANDARD: Tier = { maxEdge: 1568, maxTokens: 1568 }
12
13const PATCH = 28
14
15/**
16 * The tier for a model id or display name (`claude-haiku-4-5-20251001`,
17 * `Opus 4.7`). A name with no version, such as an alias, is taken to be a
18 * current model.
19 */
20export function tierOf(model: string): Tier {
21  const version = /(\d+)[.-](\d{1,2})(?!\d)/.exec(model)
22  if (!version) return HIGH_RES
23  const major = Number(version[1])
24  const minor = Number(version[2])
25  return major > 4 || (major === 4 && minor >= 7) ? HIGH_RES : STANDARD
26}
27
28/** Visual tokens an image costs: one per 28×28 patch. */
29export function imageTokens({ width, height }: Size): number {
30  return Math.ceil(width / PATCH) * Math.ceil(height / PATCH)
31}
32
33/**
34 * The size Claude scales an image to before reading it: unchanged when it
35 * fits the tier, else the largest aspect-preserving size that does. A port
36 * of the reference implementation in
37 * https://platform.claude.com/docs/en/build-with-claude/vision-coordinates
38 */
39export function sentSize(size: Size, tier: Tier): Size {
40  const fits = (width: number, height: number) =>
41    Math.ceil(width / PATCH) * PATCH <= tier.maxEdge &&
42    Math.ceil(height / PATCH) * PATCH <= tier.maxEdge &&
43    imageTokens({ width, height }) <= tier.maxTokens
44
45  const { width, height } = size
46  if (fits(width, height)) return size
47  if (height > width) {
48    const turned = sentSize({ width: height, height: width }, tier)
49    return { width: turned.height, height: turned.width }
50  }
51  // Binary search along the long edge: lo always fits, hi never does.
52  const aspect = width / height
53  const heightAt = (w: number) => Math.max(roundHalfEven(w / aspect), 1)
54  let lo = 1
55  let hi = width
56  while (lo + 1 < hi) {
57    const mid = Math.floor((lo + hi) / 2)
58    if (fits(mid, heightAt(mid))) lo = mid
59    else hi = mid
60  }
61  return { width: lo, height: heightAt(lo) }
62}
63
64/** Rounds to the nearest integer, ties to even, as the reference does. */
65function roundHalfEven(x: number): number {
66  const floor = Math.floor(x)
67  const fraction = x - floor
68  if (fraction !== 0.5) return Math.round(x)
69  return floor % 2 === 0 ? floor : floor + 1
70}
71

1import type { Elements, RenderElement } from 'claude-code'
2
3import type { PastedImage } from '../types'
4import { fitImage, fitThumbnails } from './fit'
5import type { Header } from './header'
6
7type Ui = Elements['terminal']
8type Band = { columns: number; rows: number }
9
10// A rounded border takes a cell on every side.
11const BORDER = 2
12
13// The header's title and details lines.
14const HEADER_ROWS = 2
15
16/**
17 * One image as large as the band allows, under its header, in a frame as
18 * wide as the wider of the two.
19 */
20export function expanded(
21  { Box, Image, Text }: Ui,
22  image: PastedImage,
23  { title, details }: Header,
24  band: Band,
25): RenderElement {
26  const cells = fitImage(image, {
27    columns: band.columns - BORDER - 2,
28    rows: band.rows - BORDER - HEADER_ROWS,
29  })
30  return (
31    <Box width={band.columns} justifyContent="center">
32      <Box
33        flexDirection="column"
34        alignItems="center"
35        paddingX={1}
36        borderStyle="round"
37        borderDimColor
38      >
39        <Text bold wrap="truncate-end">
40          {title}
41        </Text>
42        <Text dimColor wrap="truncate-end">
43          {details}
44        </Text>
45        <Image
46          key={`image-${image.n}`}
47          source={{ file: image.path, format: 'png' }}
48          columns={cells.columns}
49          rows={cells.rows}
50          alt={`[Image #${image.n}]`}
51        />
52      </Box>
53    </Box>
54  )
55}
56
57const GAP = 1
58
59/**
60 * Every image small, in one row of tiles of equal height: each picture
61 * centered in its tile, over its tag's number and caption cut to its width,
62 * so the labels line up.
63 */
64export function thumbnails(
65  { Box, Image, Text }: Ui,
66  images: readonly PastedImage[],
67  captions: ReadonlyMap<number, string>,
68  band: Band,
69): RenderElement {
70  const tiles = fitThumbnails(
71    images,
72    { columns: band.columns, rows: band.rows - BORDER - 1 },
73    BORDER + GAP,
74  )
75  return (
76    <Box columnGap={GAP}>
77      {tiles.map(({ image, cells }) => (
78        <Box
79          key={`tile-${image.n}`}
80          flexDirection="column"
81          alignItems="center"
82          borderStyle="round"
83          borderDimColor
84        >
85          <Box flexGrow={1} alignItems="center">
86            <Image
87              key={`thumbnail-${image.n}`}
88              source={{ file: image.path, format: 'png' }}
89              columns={cells.columns}
90              rows={cells.rows}
91              alt={`[Image #${image.n}]`}
92            />
93          </Box>
94          <Box width={cells.columns}>
95            <Text dimColor wrap="truncate-end">
96              {[`#${image.n}`, captions.get(image.n)].filter(Boolean).join(' ')}
97            </Text>
98          </Box>
99        </Box>
100      ))}
101    </Box>
102  )
103}
104

1import type { Size } from './png'
2
3/** A box of terminal cells. */
4export type Cells = { columns: number; rows: number }
5
6// A terminal cell is about twice as tall as it is wide.
7const CELL_ASPECT = 2
8// The Image element takes 1 to 255 cells each way.
9const MAX_CELLS = 255
10
11/** The largest box within `bounds` that keeps the picture's shape. */
12export function fitImage(size: Size, bounds: Cells): Cells {
13  const maxColumns = Math.min(bounds.columns, MAX_CELLS)
14  const maxRows = Math.min(bounds.rows, MAX_CELLS)
15  const columnsPerRow = (CELL_ASPECT * size.width) / size.height
16  const columns = Math.min(maxColumns, Math.round(maxRows * columnsPerRow))
17  const rows = Math.min(maxRows, Math.round(columns / columnsPerRow))
18  return { columns: Math.max(columns, 1), rows: Math.max(rows, 1) }
19}
20
21const THUMBNAIL_ROWS = 6
22const THUMBNAIL_MAX_COLUMNS = 30
23
24/**
25 * A thumbnail for each image in one row, all the same height and each its
26 * own width, shrunk together until the row fits `bounds.columns` with
27 * `tileColumns` more per tile for its border and gap. At one row tall they
28 * stop shrinking and the band clips what's left.
29 */
30export function fitThumbnails<T extends Size>(
31  images: readonly T[],
32  bounds: Cells,
33  tileColumns: number,
34): { image: T; cells: Cells }[] {
35  const row = (rows: number) =>
36    images.map(image => ({
37      image,
38      cells: fitImage(image, { columns: THUMBNAIL_MAX_COLUMNS, rows }),
39    }))
40  const width = (tiles: { cells: Cells }[]) =>
41    tiles.reduce((total, { cells }) => total + cells.columns + tileColumns, 0)
42
43  let rows = Math.max(Math.min(THUMBNAIL_ROWS, bounds.rows), 1)
44  while (rows > 1 && width(row(rows)) > bounds.columns) rows -= 1
45  return row(rows)
46}
47

1/** An image pasted into the prompt, as Claude Code cached it. */
2export type PastedImage = {
3  /** The number in its `[Image #n]` tag. */
4  n: number
5  /** Absolute path of the cached PNG. */
6  path: string
7  width: number
8  height: number
9  /** File size in bytes. */
10  bytes: number
11}
12
13/** What the band shows: the draft's images, and the one to show large. */
14export type Shown = {
15  images: PastedImage[]
16  /** The `n` of the image whose tag touches the cursor, or null. */
17  focused: number | null
18}
19
20/** A pasted image's caption, as the caption run left it. */
21export type Caption =
22  | { status: 'pending' }
23  | { status: 'done'; text: string }
24  | { status: 'failed' }
25
26declare module 'claude-code' {
27  interface PluginState {
28    lightbox: {
29      shown: Shown
30      /** Captions by image path. */
31      captions: Record<string, Caption>
32    }
33  }
34}
35