image-preview

1import { atom, memberOf, read, update } from 'claude-code'
2import type { Elements, EngineInterface, EventOf, Register, RenderElement } from 'claude-code'
3
4import { fit, parseBmp, toCells } from './pixels'
5import type { Pixels } from './pixels'
6import { LINUX, MACOS, WINDOWS, WINDOWS_CONVERT, powershell } from './window'
7import type { Picture, Shown } from '../types'
8
9type Engine = EngineInterface
10type RenderOf<C extends string> = Extract<EventOf['ui.render'], { component: C, surface: 'terminal' }>
11type Bounds = { columns: number; rows: number }
12type Ui = Elements['terminal']
13
14// A transcript row: a card floats over the rows above the end of one.
15type Row = RenderOf<'UserMessage' | 'AssistantMessage' | 'ToolUse' | 'ToolResult' | 'ToolGroup' | 'CommandOutput' | 'TurnDuration' | 'InfoNotice'>
16type OnScreen = NonNullable<Row['props']['onScreen']>
17
18const draft = atom({ plugin: 'image-preview', key: 'draft' } as const, [] as number[])
19const pictures = atom({ plugin: 'image-preview', key: 'pictures' } as const, {} as Record<string, Picture>)
20const shown = atom({ plugin: 'image-preview', key: 'shown' } as const, null as Shown | null)
21const hosting = atom({ plugin: 'image-preview', key: 'hosting' } as const, false)
22
23const TOKEN = /\[Image #(\d+)\]/g
24const NOT_STORED = 'not stored in this session'
25// The longest side a picture is shrunk to before it is sampled into cells, eight points a cell across.
26const SOURCE_PIXELS = 800
27const MAX_COLUMNS = 160
28const MAX_ROWS = 48
29// What the screen keeps under the transcript (the band's row, the prompt and its lines), and above
30// its first row (the logo); the fewest rows a card is worth floating in, from the band's chips and
31// from a sent prompt's.
32const BOTTOM_ROWS = 7
33const HEADER_ROWS = 4
34const MIN_FLOAT_ROWS = 10
35const MIN_CARD_ROWS = 6
36// What a card adds around the picture: the border, and the caption under it.
37const CARD_COLUMNS = 2
38const CARD_ROWS = 3
39// Under a sent prompt the engine lists its images, one a row, each after `  ⎿  `; the prompt's own
40// first line starts after `❯ `, on the row after the blank one above it.
41const LISTED_AT = 5
42const PROMPT_AT = 2
43const WAIT_MS = 200
44// Pasting an image puts its placeholder in the box with no edit event: the box is looked at this often.
45const POLL_MS = 500
46const WAIT_TRIES = 15
47// The longest side of the picture a terminal is sent to draw itself, and the most bytes it may take.
48const VIEW_PIXELS = 1024
49const VIEW_BYTES = 2 * 1024 * 1024
50// The longest side of the picture in a window of its own, in points, and the most seconds it stays.
51const WINDOW_SIZE = 800
52const WINDOW_LIFE = 120
53
54/** The ids of the `[Image #N]` placeholders in a text, each once, in order. */
55export const idsIn = (text: string) => [...new Set([...text.matchAll(TOKEN)].map(match => Number(match[1])))]
56
57const sameIds = (a: readonly number[], b: readonly number[] = []) => a.length === b.length && a.every((id, i) => id === b[i])
58
59const sizeOf = (bytes: number) =>
60  bytes < 1024 ? `${bytes} B` : bytes < 1024 * 1024 ? `${Math.round(bytes / 1024)} KB` : `${(bytes / 1024 / 1024).toFixed(1)} MB`
61
62// How the terminal draws the picture itself: Ghostty and kitty read the file; one Claude Code was
63// told draws pictures (CLAUDE_CODE_FORCE_TERMINAL_IMAGES) is sent its bytes; any other gets it in
64// colored cells.
65let graphics: 'none' | 'file' | 'bytes' = 'none'
66let sessionId: string | undefined
67let folder: string | undefined
68let uid: string | undefined
69type Platform = 'macos' | 'linux' | 'windows' | 'other'
70// The system the session runs on: what converts, opens and shows a picture there.
71let platform: Promise<Platform> | undefined
72type Tool = 'sips' | 'magick' | 'convert' | 'powershell'
73type Command = { argv: string[]; env?: Record<string, string> }
74// The converters each system is tried with, in order: sips on macOS, ImageMagick, and on Windows
75// PowerShell's System.Drawing (there `convert` is the system's disk tool, never ImageMagick's).
76const TOOLS: Record<Platform, Tool[]> = { macos: ['sips', 'magick', 'convert'], linux: ['magick', 'convert'], windows: ['magick', 'powershell'], other: ['magick', 'convert'] }
77// The converter that worked here.
78let tool: Tool | undefined
79// Where a card opens in a window of its own, by the pointer: the system whose program shows it.
80// Unset, the terminal draws the card. And the window open now, if any.
81let windowing: Platform | undefined
82let window: { id: number; isOpen: boolean } | undefined
83// The pictures read, shrunk, by id; the cells drawn of them, by id and size.
84const thumbs = new Map<number, Pixels>()
85const loading = new Set<number>()
86const cells = new Map<string, string>()
87// The PNG bytes, base64, sent to a terminal that is not known to read files, by id.
88const views = new Map<number, string>()
89// Each transcript row drawn, by requestId: which it is, when first drawn, whether in the batch the
90// session (or a reload) starts by drawing, and which of its rows show. Whether that batch is still
91// coming in. How many times a row was added or moved, and how many when the card opened. And the
92// screen's height, as the last drawing said.
93type Noted = { component: Row['component']; seen: number; isBatch: boolean; onScreen: OnScreen | null | undefined }
94const rows = new Map<string, Noted>()
95let seenRows = 0
96// The module loads as the session starts or reloads, and the engine draws every row right after.
97let isBatching = true
98let seenAtLastLook = 0
99let moves = 0
100let movesShown = 0
101let screenRows = 40
102
103/** The system the session runs on, asked once. */
104function platformOf ($: Engine) {
105  platform ??= (async (): Promise<Platform> => {
106    if ((await $.env.get('OS')) === 'Windows_NT') return 'windows'
107    const name = (await $.process.run(['uname', '-s']).catch(() => undefined))?.stdout.trim()
108    return name === 'Darwin' ? 'macos' : name === 'Linux' ? 'linux' : 'other'
109  })()
110  return platform
111}
112
113/** Forgets what was read in another session: after /clear the ids start again from 1. */
114async function follow ($: Engine) {
115  const id = await $.session.id()
116  if (id === sessionId) return
117  // The rows drawn so far are this session's, unless another came before it.
118  if (sessionId !== undefined) rows.clear()
119  sessionId = id
120  folder = undefined
121  thumbs.clear()
122  cells.clear()
123  views.clear()
124  await update($, pictures, () => ({}))
125  await change($, () => null)
126}
127
128/**
129 * Claude Code's temp folder for this session, where a pasted image is stored as
130 * `images/<N>.<ext>`: `<CLAUDE_CODE_TMPDIR or /tmp>/claude-<uid>/<project path, dashed>/<session id>`.
131 */
132async function sessionFolder ($: Engine) {
133  if (folder) return folder
134  const id = await $.session.id()
135  const base = ((await $.env.get('CLAUDE_CODE_TMPDIR')) || '/tmp').replace(/\/+$/, '')
136  // Windows has no uid: Claude Code names its folder `claude-0` there.
137  uid ??= (await platformOf($)) === 'windows' ? '0' : (await $.process.run(['id', '-u'])).stdout.trim()
138  const root = `${base}/claude-${uid}`
139  const names = new Set([await $.session.root(), await $.session.cwd()].map(path => path.replace(/[^a-zA-Z0-9]/g, '-')))
140  for (const name of names) {
141    if (await $.fs.exists(`${root}/${name}/${id}/images`)) return (folder = `${root}/${name}/${id}`)
142  }
143  // A project path too long for a folder name gets a hash after it: look for the session instead.
144  for (const entry of await $.fs.list(root).catch(() => [])) {
145    if (entry.kind === 'dir' && (await $.fs.exists(`${root}/${entry.name}/${id}/images`))) {
146      return (folder = `${root}/${entry.name}/${id}`)
147    }
148  }
149  return undefined
150}
151
152async function fileOf ($: Engine, id: number) {
153  const found = await sessionFolder($)
154  if (!found) return undefined
155  const name = (await $.fs.list(`${found}/images`).catch(() => [])).find(entry => entry.name.startsWith(`${id}.`))?.name
156  return name && /^\d+\.[a-z]+$/.test(name) ? `${found}/images/${name}` : undefined
157}
158
159/** Runs the first converter that works here, and remembers it. */
160async function convert ($: Engine, commandOf: (tool: Tool) => Command) {
161  for (const name of tool ? [tool] : TOOLS[await platformOf($)]) {
162    const { argv, env } = commandOf(name)
163    const result = await $.process.run(argv, env && { env }).catch(() => undefined)
164    if (result?.exitCode === 0) {
165      tool = name
166      return true
167    }
168  }
169  return false
170}
171
172/**
173 * How `tool` writes `file` as `format` to `out`, its longest side `side` pixels (`n` stretched or
174 * shrunk to it, `n>` only shrunk; none, as it is).
175 */
176function shrink (tool: Tool, file: string, out: string, format: 'bmp' | 'png', side?: string): Command {
177  if (tool === 'sips') return { argv: ['sips', '-s', 'format', format, ...(side ? ['-Z', side.replace('>', '')] : []), file, '--out', out] }
178  if (tool === 'powershell') {
179    return { argv: powershell(WINDOWS_CONVERT), env: { IMAGE_PREVIEW_FILE: file, IMAGE_PREVIEW_OUT: out, IMAGE_PREVIEW_FORMAT: format, ...(side && { IMAGE_PREVIEW_SIZE: side.replace('>', '') }) } }
180  }
181  const size = side && `${side.replace('>', '')}x${side}`
182  return { argv: [tool, file, ...(size ? ['-resize', size] : []), `${format === 'bmp' ? 'BMP3' : 'PNG'}:${out}`] }
183}
184
185/** Makes the mod's cache folder. */
186async function makeFolder ($: Engine, path: string) {
187  if ((await platformOf($)) !== 'windows') return void (await $.process.run(['mkdir', '-p', path]))
188  await $.process.run(powershell('New-Item -ItemType Directory -Force -Path $env:IMAGE_PREVIEW_OUT | Out-Null'), { env: { IMAGE_PREVIEW_OUT: path } })
189}
190
191async function decode ($: Engine, file: string, out: string): Promise<Pixels> {
192  const isConverted = await convert($, name => shrink(name, file, out, 'bmp', String(SOURCE_PIXELS)))
193  if (!isConverted) throw new Error('needs sips (macOS), ImageMagick or PowerShell (Windows) to read it')
194  const { base64 } = await $.fs.read(out, { as: 'bytes' })
195  return parseBmp(Uint8Array.fromBase64(base64))
196}
197
198/** A PNG of the picture for the terminal to draw itself: the file, or a copy converted beside the cache. */
199async function pngOf ($: Engine, file: string, out: string) {
200  if (file.endsWith('.png')) return file
201  return (await convert($, name => shrink(name, file, out, 'png'))) ? out : undefined
202}
203
204/** The picture shrunk to a PNG, base64, for a terminal sent the bytes: none past what one may take. */
205async function viewOf ($: Engine, file: string, out: string) {
206  const isConverted = await convert($, name => shrink(name, file, out, 'png', `${VIEW_PIXELS}>`))
207  if (!isConverted || (await $.fs.stat(out)).size > VIEW_BYTES) return undefined
208  return (await $.fs.read(out, { as: 'bytes' })).base64
209}
210
211/** Reads image `id` once: its file, then its pixels, shrunk; the drawings that wait on it redraw. */
212async function load ($: Engine, id: number) {
213  if (thumbs.has(id) || loading.has(id)) return
214  loading.add(id)
215  let picture: Picture
216  try {
217    // The engine writes the file just after the paste puts the placeholder in the box.
218    let file = await fileOf($, id)
219    for (let tries = 0; !file && tries < WAIT_TRIES; tries++) {
220      await $.clock.sleep(WAIT_MS)
221      file = await fileOf($, id)
222    }
223    if (!file) throw new Error(NOT_STORED)
224
225    const cache = `${await sessionFolder($)}/image-preview`
226    await makeFolder($, cache)
227    const pixels = await decode($, file, `${cache}/${id}.bmp`)
228    let png: string | undefined
229    if (graphics === 'file') png = await pngOf($, file, `${cache}/${id}.png`)
230    if (graphics === 'bytes') {
231      const view = await viewOf($, file, `${cache}/${id}.view.png`)
232      if (view) {
233        views.set(id, view)
234        png = `${cache}/${id}.view.png`
235      }
236    }
237    const { size } = await $.fs.stat(file)
238    thumbs.set(id, pixels)
239    const format = file.slice(file.lastIndexOf('.') + 1)
240    picture = { file, format, bytes: size, width: pixels.width, height: pixels.height, ...(png && { png }) }
241  } catch (error) {
242    picture = { error: error instanceof Error ? error.message : String(error) }
243  } finally {
244    loading.delete(id)
245  }
246  // Read in the background: a module unloaded meanwhile has no state to write to.
247  await update($, pictures, all => ({ ...all, [id]: picture })).catch(() => undefined)
248}
249
250async function fileFor ($: Engine, id: number) {
251  const picture = (await read($, pictures))[id]
252  return picture && !('error' in picture) ? picture.file : await fileOf($, id)
253}
254
255/** Opens image `id` in the system's viewer: Preview on macOS, the desktop's own elsewhere. */
256async function openImage ($: Engine, id: number) {
257  const file = await fileFor($, id)
258  if (!file) return
259  const system = await platformOf($)
260  const argv = system === 'macos' ? ['open', '-a', 'Preview', file] : system === 'windows' ? ['explorer', file.replaceAll('/', '\\')] : ['xdg-open', file]
261  await $.process.run(argv).catch(() => undefined)
262}
263
264/** The program that shows image `id` in a window of its own here, and what it is told. */
265async function windowCommand ($: Engine, id: number): Promise<Command | undefined> {
266  const file = await fileFor($, id)
267  if (!file) return undefined
268  const env = { IMAGE_PREVIEW_FILE: file, IMAGE_PREVIEW_SIZE: String(WINDOW_SIZE), IMAGE_PREVIEW_LIFE: String(WINDOW_LIFE) }
269  if (windowing === 'macos') return { argv: ['osascript', '-l', 'JavaScript', '-e', MACOS], env }
270  if (windowing === 'windows') return { argv: powershell(WINDOWS), env }
271  // Tk reads a PNG: the picture shrunk to one, once.
272  const cache = `${await sessionFolder($)}/image-preview`
273  const out = `${cache}/${id}.window.png`
274  if (!(await $.fs.exists(out))) {
275    await makeFolder($, cache)
276    if (!(await convert($, name => shrink(name, file, out, 'png', `${WINDOW_SIZE}>`)))) return undefined
277  }
278  return { argv: ['python3', '-c', LINUX], env: { ...env, IMAGE_PREVIEW_FILE: out } }
279}
280
281/**
282 * Shows image `id` in a window of its own till `closeWindow`. The program says `.` ten times a
283 * second, so the loop wakes to see it was closed, and leaving the loop ends the program. One that
284 * ends without having shown a window says the system shows none: the card goes back to the terminal.
285 */
286async function openWindow ($: Engine, id: number) {
287  if (window?.id === id && window.isOpen) return
288  closeWindow()
289  const opening = { id, isOpen: true }
290  window = opening
291  const command = await windowCommand($, id)
292  let isShown = false
293  let said = ''
294  if (command && opening.isOpen) {
295    try {
296      for await (const { text } of $.process.spawn(command)) {
297        if (said.length < 2000) said += text
298        if (/^shown$/m.test(text)) isShown = true
299        if (!opening.isOpen) break
300      }
301    } catch (error) {
302      said += error instanceof Error ? error.message : String(error)
303    }
304  }
305  if (window === opening) window = undefined
306  if (isShown || !opening.isOpen) return
307  $.ui.log(`image-preview: no window (${said.replace(/\.\n/g, '').trim().slice(0, 300) || 'the program could not start'}); the terminal draws the cards`, { to: 'debug' })
308  windowing = undefined
309  await change($, now => (now?.edge === 'window' ? placeFor(now.id, now.target) : now))
310}
311
312function closeWindow () {
313  if (window) window.isOpen = false
314  window = undefined
315}
316
317/**
318 * The picture of image `id` in a bordered card, as large as the bounds allow, its caption under it.
319 * It floats over the transcript, which shows through any cell it leaves unwritten: every line is
320 * written to the card's width, spaces and all.
321 */
322function card ({ Box, Image, Raster, Text }: Ui, all: Record<string, Picture>, id: number, bounds: Bounds) {
323  const label = `[Image #${id}]`
324  const picture = all[id]
325  const line = (text: string, width: number) => <Text dimColor wrap="truncate-end">{text.padEnd(width)}</Text>
326  if (!picture || (!('error' in picture) && !thumbs.has(id))) return line(`${label} loading…`, bounds.columns)
327  if ('error' in picture) return line(`${label} no preview: ${picture.error}`, bounds.columns)
328
329  const { columns, rows } = fit(picture.width, picture.height, bounds.columns - CARD_COLUMNS, bounds.rows - CARD_ROWS)
330  const view = views.get(id)
331  const source = !picture.png ? undefined : graphics === 'file' ? { file: picture.png, format: 'png' as const } : view && { png: view }
332  const key = `${id}:${columns}x${rows}`
333  let painted = cells.get(key)
334  if (!source && painted === undefined) {
335    painted = toCells(thumbs.get(id)!, columns, rows)
336    cells.set(key, painted)
337  }
338  const caption = `${label} ${picture.format.toUpperCase()} · ${sizeOf(picture.bytes)}`
339  // A picture narrower than its caption is centered, the rows beside it written blank.
340  const width = Math.max(columns, Math.min(caption.length, bounds.columns - CARD_COLUMNS))
341  const side = (n: number) => n > 0 && <Text>{Array.from({ length: rows }, () => ' '.repeat(n)).join('\n')}</Text>
342  const left = Math.floor((width - columns) / 2)
343  return (
344    <Box key={`card-${id}`} flexDirection="column" borderStyle="round" borderDimColor flexShrink={0} width={width + CARD_COLUMNS}>
345      <Box flexDirection="row">
346        {side(left)}
347        {source
348          ? <Image key={`image-${id}`} source={source} columns={columns} rows={rows} alt={label} />
349          : <Raster key={`image-${id}`} columns={columns} rows={rows} cells={painted!} />}
350        {side(width - columns - left)}
351      </Box>
352      {line(caption, width)}
353    </Box>
354  )
355}
356
357const sameShown = (a: Shown | null, b: Shown | null) =>
358  a === b || (a !== null && b !== null && a.id === b.id && a.target === b.target && a.host === b.host && a.rows === b.rows && a.edge === b.edge)
359
360let changing = Promise.resolve()
361
362/**
363 * Opens, moves or closes (null) the card, one change at a time, each `next` handed the card the one
364 * before left: the pointer leaving one chip and coming onto the next are two changes, in either
365 * order. The row the card leaves and the one it joins draw again.
366 */
367function change ($: Engine, next: (now: Shown | null) => Shown | null) {
368  const run = changing.then(async () => {
369    const last = await read($, shown)
370    const to = next(last)
371    if (sameShown(last, to)) return
372    movesShown = moves
373    await update($, shown, () => to)
374    if (last?.host && last.host !== to?.host) await update($, memberOf(hosting, { requestId: last.host }), () => false)
375    if (to?.host && to.host !== last?.host) await update($, memberOf(hosting, { requestId: to.host }), () => true)
376    if (to?.edge === 'window') void openWindow($, to.id)
377    else closeWindow()
378  })
379  changing = run.catch(() => undefined)
380  return run
381}
382
383/** Closes the card a chip of the band opened, if one did. */
384const closeChipCard = (now: Shown | null) => (now && !isFromPrompt(now) ? null : now)
385
386const isFromPrompt = (now: Shown) => /\/(sent|token):/.test(now.target)
387
388/** Where the card a target opened goes: a target is `<site>/<chip key>`, the site a sent prompt's for its chips. */
389const placeFor = (id: number, target: string) => placeOf(id, target, /\/chip:/.test(target) ? undefined : target.slice(0, target.indexOf('/')))
390
391/** Notes a transcript row, and which of its rows show: before anything is awaited, so in the transcript's order. */
392function note (e: Row) {
393  if (e.viewport) screenRows = e.viewport.rows
394  const onScreen = e.props.onScreen
395  const row = rows.get(e.requestId)
396  if (!row) {
397    rows.set(e.requestId, { component: e.component, seen: seenRows++, isBatch: isBatching, onScreen })
398    moves++
399  } else if (row.onScreen?.first !== onScreen?.first || row.onScreen?.last !== onScreen?.last || row.onScreen?.of !== onScreen?.of) {
400    row.onScreen = onScreen
401    moves++
402  }
403}
404
405/**
406 * The rows in the transcript's order, as best known. A row drawn as it came is in place. The batch
407 * a start or a reload draws comes in two runs, the sent prompts apart from the rest (each run in
408 * order): each prompt is put back at the start of a turn, the row after a turn's duration line,
409 * the newest prompt at the newest turn.
410 */
411function transcriptOrder () {
412  const bySeen = (a: [string, Noted], b: [string, Noted]) => a[1].seen - b[1].seen
413  const batch = [...rows].filter(([, row]) => row.isBatch).sort(bySeen)
414  const notices = batch.filter(([, row]) => row.component === 'InfoNotice')
415  const prompts = batch.filter(([, row]) => row.component === 'UserMessage')
416  const others = batch.filter(([, row]) => row.component !== 'InfoNotice' && row.component !== 'UserMessage')
417  const starts = [0, ...others.flatMap(([, row], i) => (row.component === 'TurnDuration' && i + 1 < others.length ? [i + 1] : []))]
418  const before = new Map<number, string[]>()
419  prompts.forEach(([id], i) => {
420    const at = starts[Math.max(0, starts.length - (prompts.length - i))]!
421    before.set(at, [...(before.get(at) ?? []), id])
422  })
423  const ordered = [...notices.map(([id]) => id), ...others.flatMap(([id], i) => [...(before.get(i) ?? []), id])]
424  if (others.length === 0) ordered.push(...prompts.map(([id]) => id))
425  const order = new Map(ordered.map((id, i) => [id, i]))
426  for (const [id, row] of rows) if (!row.isBatch) order.set(id, ordered.length + row.seen)
427  return order
428}
429
430/**
431 * Where the card of image `id` floats, from what the transcript's rows said of themselves; it never
432 * covers the chip that opened it, and covers only rows drawn before the one it stands on (a row drawn
433 * after paints over it).
434 *
435 * From the band's chip: over the rows above the end of the lowest row on screen; in the band
436 * itself where too few rows show. From the list under a sent prompt (`prompt`, its requestId): over
437 * the rows above the prompt, or over those under its list, whichever holds more; none where neither
438 * holds a card. The rows counted are the rows the mod knows of, so a count is never too high.
439 */
440function placeOf (id: number, target: string, prompt?: string): Shown | null {
441  if (windowing) return { id, target, host: null, rows: 0, edge: 'window' }
442  const order = transcriptOrder()
443  const showing = [...rows]
444    .filter(([, row]) => !!row.onScreen)
445    .map(([id, row]) => [id, { order: order.get(id)!, onScreen: row.onScreen! }] as const)
446  // A row first drawn late (as one scrolled into view) still has its place where it is cut: the row
447  // cut at the top is the first on screen, the one cut at the foot the last.
448  const placeOn = ({ order, onScreen }: { order: number; onScreen: OnScreen }) =>
449    onScreen.first > 0 ? -Infinity : onScreen.last < onScreen.of - 1 ? Infinity : order
450  showing.sort((a, b) => placeOn(a[1]) - placeOn(b[1]) || a[1].order - b[1].order)
451  const room = screenRows - BOTTOM_ROWS
452  const lowest = showing.at(-1)
453  const top = showing[0]?.[1]
454  // A row off the screen, or the first on it cut, is a transcript that fills the screen.
455  const isFull = top !== undefined && (top.onScreen.first > 0 || [...rows.values()].some(row => row.onScreen === null))
456  const count = (list: typeof showing) => list.reduce((sum, [, row]) => sum + row.onScreen.last - row.onScreen.first + 1, 0)
457
458  if (prompt === undefined) {
459    // One row kept free: the count of the rows above is a guess.
460    const above = !lowest ? 0 : (isFull ? room : Math.min(room, HEADER_ROWS + count(showing))) - 1
461    return above >= MIN_FLOAT_ROWS
462      ? { id, target, host: lowest![0], rows: above, edge: 'last' }
463      : { id, target, host: null, rows: 0, edge: 'last' }
464  }
465  const at = showing.findIndex(([requestId]) => requestId === prompt)
466  if (at === -1) return null
467  const message = showing[at]![1].onScreen
468  // Above: the rows before it that show, the logo's in a short transcript, and its own blank first row.
469  const above = message.first > 0 ? 0 : Math.min(room, count(showing.slice(0, at)) + (isFull ? 0 : HEADER_ROWS) + 1)
470  // Under: the rows after it, between its list and the foot of the lowest.
471  const under = Math.min(room, count(showing.slice(at + 1)))
472  if (Math.max(above, under) < MIN_CARD_ROWS) return null
473  return above >= under
474    ? { id, target, host: prompt, rows: above, edge: 'first' }
475    : { id, target, host: lowest![0], rows: under, edge: 'last' }
476}
477
478/**
479 * Keeps the open card in step with the transcript. A sent prompt's list moves with it, and a chip
480 * that moves from under a resting pointer says nothing: its card closes when anything moves (a move
481 * of the pointer opens it again). The band's chips stay put: their card follows the lowest row.
482 */
483async function settle ($: Engine) {
484  if (seenRows === seenAtLastLook) isBatching = false
485  seenAtLastLook = seenRows
486  await change($, now => !now ? null : !isFromPrompt(now) ? placeFor(now.id, now.target) : moves === movesShown ? now : null)
487}
488
489/** A transcript row, drawn with the open card where it holds it: out of the flow, so it moves nothing. */
490async function float ($: Engine, e: Row, drawn: RenderElement, overlays: RenderElement[] = []) {
491  const now = (await read($, memberOf(hosting, e))) ? await read($, shown) : null
492  const onScreen = e.props.onScreen
493  const isHost = now !== null && now.host === e.requestId && !!onScreen
494  if (!isHost && overlays.length === 0) return drawn
495  const ui = $.ui.resolve(e)
496  const { Box } = ui
497  let floating: RenderElement | undefined
498  if (isHost) {
499    const bounds = { columns: Math.max(8, Math.min(MAX_COLUMNS, (e.viewport?.columns ?? 80) - 4)), rows: Math.min(MAX_ROWS, now.rows) }
500    // On the last row of this one that shows (the rest are past the screen's edge), or ending on
501    // its first, the blank row above a message.
502    const bottom = now.edge === 'last' ? onScreen.of - 1 - onScreen.last : onScreen.of - 1
503    floating = (
504      <Box key={`float-${now.id}`} position="absolute" bottom={bottom} left={0}>
505        {card(ui, await read($, pictures), now.id, bounds)}
506      </Box>
507    )
508  }
509  return (
510    <Box flexDirection="column">
511      {drawn}
512      {overlays}
513      {floating}
514    </Box>
515  )
516}
517
518/** Keeps the band's list of the draft's images in step with the prompt box. */
519async function track ($: Engine, text: string) {
520  await follow($)
521  const ids = idsIn(text)
522  if (!sameIds(ids, await read($, draft))) {
523    await update($, draft, () => ids)
524    await change($, closeChipCard)
525  }
526  for (const id of ids) void load($, id)
527}
528
529export const register: Register = (on, options) => {
530  on('session.start', async ($, e, next) => {
531    const result = await next(e)
532    const program = await $.env.get('TERM_PROGRAM')
533    const term = await $.env.get('TERM')
534    const isNative = program === 'ghostty' || term === 'xterm-ghostty' || term === 'xterm-kitty' || (await $.env.get('KITTY_WINDOW_ID')) !== undefined
535    const isForced = /^(1|true|yes|on)$/i.test((await $.env.get('CLAUDE_CODE_FORCE_TERMINAL_IMAGES')) ?? '')
536    graphics = isNative ? 'file' : isForced ? 'bytes' : 'none'
537    // Where the terminal cannot draw the picture itself, a card opens in a window of its own, by
538    // the pointer, on the desktop the session runs on: not over SSH, where that is another machine's.
539    windowing = undefined
540    const isRemote = (await $.env.get('SSH_CONNECTION')) !== undefined || (await $.env.get('SSH_TTY')) !== undefined
541    if (options.window !== false && graphics === 'none' && e.isInteractive && !isRemote) {
542      const system = await platformOf($)
543      if (system === 'macos' || system === 'windows') windowing = system
544      if (system === 'linux' && (await $.env.get('DISPLAY'))) {
545        const tk = await $.process.run(['python3', '-c', 'import tkinter']).catch(() => undefined)
546        if (tk?.exitCode === 0) windowing = 'linux'
547      }
548    }
549    // A reload starts with an empty cache: read again what is in the box.
550    await follow($)
551    await track($, (await $.prompt.read()).text)
552    if (e.isInteractive) {
553      // A look that fails (the module unloading under it) waits for the next.
554      $.clock.every(POLL_MS, () => {
555        void $.prompt.read().then(box => track($, box.text)).then(() => settle($)).catch(() => undefined)
556      })
557    }
558    return result
559  })
560
561  // Typing closes the card a chip opened: the box may grow and move the chip from under the pointer.
562  on('prompt.edit', async ($, e, next) => {
563    const box = await next(e)
564    await change($, closeChipCard)
565    await track($, box.text)
566    return box
567  })
568
569  on('prompt.submit', async ($, e, next) => {
570    const result = await next(e)
571    if (e.origin.kind !== 'composer') return result
572    await update($, draft, () => [])
573    await change($, () => null)
574    for (const id of idsIn(e.text)) void load($, id)
575    return result
576  }).catch(($, e, next) => next(e))
577
578  // A chip says when the pointer comes onto it (or moves on it) and when it leaves: its card opens
579  // and closes. Clicked, it opens the image in the system's viewer.
580  on('ui.message', async ($, e, next) => {
581    const result = await next(e)
582    const chip = /^(chip|sent|token):(\d+)$/.exec(e.element)
583    if (!chip) return result
584    const target = `${e.requestId}/${e.element}`
585    const data = e.data as { isOver?: unknown; isClicked?: unknown } | null
586    // Clicked, the image opens in its viewer, over which a card left open would stand.
587    if (data?.isClicked === true) {
588      void openImage($, Number(chip[2]))
589      await change($, () => null)
590      return result
591    }
592    const isOver = data?.isOver === true
593    await change($, now => (!isOver ? (now?.target === target ? null : now) : now?.target === target ? now : placeFor(Number(chip[2]), target)))
594    return result
595  })
596
597  // Every transcript row is noted in one hook, so rows drawn again together (as after a reload) are
598  // noted in the transcript's order. A sent prompt: under it the engine lists its images, `⎿ [Image #N]`
599  // a row; each gets a chip of its own over it, unpainted, so the engine's row shows as it was and
600  // the pointer on it opens the card.
601  on('ui.render', { component: ['UserMessage', 'AssistantMessage', 'ToolUse', 'ToolResult', 'ToolGroup', 'CommandOutput', 'TurnDuration', 'InfoNotice'] }, async ($, e, next) => {
602    if (e.surface !== 'terminal' || e.viewport?.isFullscreen === false) return next(e)
603    note(e)
604    if (e.component !== 'UserMessage') return float($, e, await next(e))
605    const all = await read($, pictures)
606    const ids = e.props.origin.kind === 'composer' ? idsIn(e.props.text) : []
607    for (const id of ids) if (!all[id]) void load($, id)
608    const drawn = await next(e)
609    // The engine lists the images it was sent, in the order pasted: each one stored, read or not.
610    const listed = ids.filter(id => { const picture = all[id]; return picture && !('error' in picture && picture.error === NOT_STORED) })
611    listed.sort((a, b) => a - b)
612    const { Box, Client } = $.ui.resolve(e)
613    const overlays: RenderElement[] = []
614    listed.forEach((id, i) => {
615      if ('error' in all[id]!) return
616      overlays.push(
617        <Box key={`listed-${id}`} position="absolute" bottom={-1 - i} left={LISTED_AT}>
618          <Client key={`sent:${id}`} module="./chip.tsx" width={`[Image #${id}]`.length} height={1} />
619        </Box>
620      )
621    })
622    // The rows the engine lists the images in take the pointer but not a click, which stays the
623    // engine's: a click opens an image from its `[Image #N]` in the prompt's first line, where the
624    // place of each can be told (before the line wraps, after text one cell a character).
625    const line = e.props.text.split('\n')[0]!
626    const width = (e.viewport?.columns ?? 80) - 1
627    const placed = new Set<number>()
628    for (const match of line.matchAll(TOKEN)) {
629      const id = Number(match[1])
630      const at = PROMPT_AT + match.index
631      const picture = all[id]
632      if (placed.has(id) || !picture || 'error' in picture || !/^[\u0020-\u02ff]*$/.test(line.slice(0, match.index)) || at + match[0].length > width) continue
633      placed.add(id)
634      overlays.push(
635        <Box key={`token-${id}`} position="absolute" top={1} left={at}>
636          <Client key={`token:${id}`} module="./chip.tsx" width={match[0].length} height={1} />
637        </Box>
638      )
639    }
640    return float($, e, drawn, overlays)
641  })
642
643  // Above the prompt: the draft's images, each a chip that opens its card while the pointer is on
644  // it. The card floats over the transcript; where too few of its rows show, the band holds it.
645  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
646    // Off fullscreen the terminal reports no pointer: nothing could be hovered.
647    if (e.surface !== 'terminal' || e.props.hasSurvey || e.viewport?.isFullscreen === false) return next(e)
648    const ids = await read($, draft)
649    const now = await read($, shown)
650    const inBand = now && now.host === null && now.edge !== 'window' ? now.id : undefined
651    if (ids.length === 0 && inBand === undefined) return next(e)
652    if (e.viewport) screenRows = e.viewport.rows
653
654    const ui = $.ui.resolve(e)
655    const { Box, Client, Text } = ui
656    const below: RenderElement = await next(e)
657    const bounds = {
658      columns: Math.max(8, Math.min(MAX_COLUMNS, e.props.bodyColumns)),
659      rows: Math.max(4, Math.min(MAX_ROWS, e.props.maxRows - 1))
660    }
661    return (
662      <Box flexDirection="column">
663        {below}
664        {inBand !== undefined && card(ui, await read($, pictures), inBand, bounds)}
665        {ids.length > 0 && (
666          <Box flexDirection="row" columnGap={1}>
667            {ids.map(id => <Client key={`chip:${id}`} module="./chip.tsx" props={{ label: `[Image #${id}]` }} />)}
668            <Text dimColor>hover to preview · click to open</Text>
669          </Box>
670        )}
671      </Box>
672    )
673  })
674}
675

1/** A picture decoded to 8-bit RGBA, row-major from the top-left pixel. */
2export type Pixels = { width: number; height: number; data: Uint8Array }
3
4const UPPER_HALF = 0x2580 // ▀
5const LEFT_HALF = 0x258c // ▌
6const SPACE = 0x20
7const DEFAULT_COLOR = 0x01000000
8
9/** The lowest set bit of a channel mask, and how many bits wide the channel is. */
10function channelOf (mask: number) {
11  if (mask === 0) return null
12  let shift = 0
13  while (((mask >>> shift) & 1) === 0) shift++
14  let bits = 0
15  while (((mask >>> (shift + bits)) & 1) === 1) bits++
16  return { shift, max: 2 ** bits - 1 }
17}
18
19/** Reads an uncompressed 24- or 32-bit BMP, the kind `sips` and ImageMagick write. */
20export function parseBmp (bytes: Uint8Array): Pixels {
21  if (bytes.length < 54 || bytes[0] !== 0x42 || bytes[1] !== 0x4d) throw new Error('not a BMP file')
22  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
23  const offset = view.getUint32(10, true)
24  const headerSize = view.getUint32(14, true)
25  const width = view.getInt32(18, true)
26  const signedHeight = view.getInt32(22, true)
27  const bits = view.getUint16(28, true)
28  const compression = view.getUint32(30, true)
29  const height = Math.abs(signedHeight)
30
31  if (width <= 0 || height === 0) throw new Error('an empty BMP')
32  if (bits !== 24 && bits !== 32) throw new Error(`a ${bits}-bit BMP`)
33  if (compression !== 0 && compression !== 3) throw new Error('a compressed BMP')
34
35  const stride = Math.ceil((width * bits) / 32) * 4
36  if (offset + stride * height > bytes.length) throw new Error('a truncated BMP')
37
38  // BI_BITFIELDS keeps its masks right after the 40-byte header, or inside a longer one.
39  const masks = compression === 3
40    ? [54, 58, 62, 66].map(at => (at === 66 && headerSize < 56) || at + 4 > bytes.length ? 0 : view.getUint32(at, true))
41    : bits === 32 ? [0xff0000, 0xff00, 0xff, 0xff000000] : null
42  const channels = masks?.map(channelOf)
43
44  const data = new Uint8Array(width * height * 4)
45  let hasAlpha = false
46  for (let y = 0; y < height; y++) {
47    const start = offset + (signedHeight < 0 ? y : height - 1 - y) * stride
48    for (let x = 0; x < width; x++) {
49      const out = (y * width + x) * 4
50      if (channels) {
51        const value = view.getUint32(start + x * 4, true)
52        for (let c = 0; c < 4; c++) {
53          const channel = channels[c]
54          data[out + c] = channel ? Math.round((((value >>> channel.shift) & channel.max) * 255) / channel.max) : 255
55        }
56        if (data[out + 3] !== 0) hasAlpha = true
57      } else {
58        const at = start + x * 3
59        data[out] = bytes[at + 2]!
60        data[out + 1] = bytes[at + 1]!
61        data[out + 2] = bytes[at]!
62        data[out + 3] = 255
63      }
64    }
65  }
66  // A writer that leaves the alpha byte unused fills it with zeros: the picture is opaque.
67  if (channels && !hasAlpha) for (let i = 3; i < data.length; i += 4) data[i] = 255
68
69  return { width, height, data }
70}
71
72/**
73 * The largest box of cells within the bounds that keeps the picture's shape. A cell is about
74 * twice as tall as it is wide, and holds two rows of pixels.
75 */
76export function fit (width: number, height: number, maxColumns: number, maxRows: number) {
77  const columnsCap = Math.max(1, Math.min(maxColumns, width))
78  let columns = columnsCap
79  let rows = Math.max(1, Math.round((columns * height) / width / 2))
80  if (rows > maxRows) {
81    rows = Math.max(1, maxRows)
82    columns = Math.max(1, Math.min(columnsCap, Math.round((rows * 2 * width) / height)))
83  }
84  return { columns, rows }
85}
86
87// Light adds up in linear terms, not in the gamma-encoded bytes: averaging the bytes darkens a
88// light line on a dark ground until it is gone.
89const LINEAR = Float64Array.from({ length: 256 }, (_, i) => {
90  const value = i / 255
91  return value <= 0.04045 ? value / 12.92 : ((value + 0.055) / 1.055) ** 2.4
92})
93const ENCODED = Uint8Array.from({ length: 4096 }, (_, i) => {
94  const value = i / 4095
95  return Math.round(255 * (value <= 0.0031308 ? value * 12.92 : 1.055 * value ** (1 / 2.4) - 0.055))
96})
97const encode = (linear: number) => ENCODED[Math.min(4095, Math.max(0, Math.round(linear * 4095)))]!
98
99// Each cell is sampled as a grid of SIDE by SIDE points, each the average of the pixels it covers.
100const SIDE = 8
101const HALF = SIDE / 2
102const POINTS = SIDE * SIDE
103
104/** A block element, and which of a cell's points its foreground covers. */
105type Shape = { glyph: number; inside: Uint8Array }
106
107const shape = (glyph: number, covers: (x: number, y: number) => boolean): Shape => ({
108  glyph,
109  inside: Uint8Array.from({ length: POINTS }, (_, i) => (covers(i % SIDE, Math.floor(i / SIDE)) ? 1 : 0))
110})
111
112// The ways a cell can be split in two colors: the halves, the eighths from the bottom and from the
113// left (each also the rest from the top or right, the colors swapped), the quarters and the
114// diagonals. The halves first, so a cell they draw as well as another is drawn with them.
115const SHAPES: Shape[] = [
116  shape(UPPER_HALF, (_, y) => y < HALF),
117  shape(LEFT_HALF, x => x < HALF),
118  ...[1, 2, 3, 5, 6, 7].map(k => shape(0x2580 + k, (_, y) => y >= SIDE - k)), // ▁▂▃▅▆▇
119  ...[1, 2, 3, 5, 6, 7].map(k => shape(0x2590 - k, x => x < k)), // ▏▎▍▋▊▉
120  shape(0x2598, (x, y) => x < HALF && y < HALF), // ▘
121  shape(0x259d, (x, y) => x >= HALF && y < HALF), // ▝
122  shape(0x2596, (x, y) => x < HALF && y >= HALF), // ▖
123  shape(0x2597, (x, y) => x >= HALF && y >= HALF), // ▗
124  shape(0x259a, (x, y) => (x < HALF) === (y < HALF)), // ▚
125  shape(0x259e, (x, y) => (x < HALF) !== (y < HALF)) // ▞
126]
127
128/** One cell's points: gamma-encoded, to choose a shape by; linear, to mix its colors in; and which show. */
129type Points = { encoded: Float64Array; linear: Float64Array; shows: Uint8Array; shown: number }
130
131/** Averages, in linear light, the pixels under each of the points of the cell at `column`, `row`. */
132function pointsOf ({ width, height, data }: Pixels, columns: number, rows: number, column: number, row: number, points: Points) {
133  const across = columns * SIDE
134  const down = rows * SIDE
135  points.shown = 0
136  for (let py = 0; py < SIDE; py++) {
137    const y0 = Math.floor(((row * SIDE + py) * height) / down)
138    const y1 = Math.max(y0 + 1, Math.floor(((row * SIDE + py + 1) * height) / down))
139    for (let px = 0; px < SIDE; px++) {
140      const x0 = Math.floor(((column * SIDE + px) * width) / across)
141      const x1 = Math.max(x0 + 1, Math.floor(((column * SIDE + px + 1) * width) / across))
142      let r = 0
143      let g = 0
144      let b = 0
145      let alpha = 0
146      for (let y = y0; y < y1; y++) {
147        for (let x = x0; x < x1; x++) {
148          const at = (y * width + x) * 4
149          const a = data[at + 3]!
150          r += LINEAR[data[at]!]! * a
151          g += LINEAR[data[at + 1]!]! * a
152          b += LINEAR[data[at + 2]!]! * a
153          alpha += a
154        }
155      }
156      const i = py * SIDE + px
157      const shows = alpha / ((y1 - y0) * (x1 - x0)) >= 128
158      points.shows[i] = shows ? 1 : 0
159      if (!shows) continue
160      points.shown++
161      points.linear[i * 3] = r / alpha
162      points.linear[i * 3 + 1] = g / alpha
163      points.linear[i * 3 + 2] = b / alpha
164      for (let c = 0; c < 3; c++) points.encoded[i * 3 + c] = encode(points.linear[i * 3 + c]!)
165    }
166  }
167}
168
169/**
170 * The shape that splits the points in the two groups closest to their own averages (least squared
171 * error, measured on the encoded values, which is closer to what the eye sees); none where one
172 * color is as close. Part see-through, the shape that best parts the points that show from the rest.
173 */
174function shapeOf ({ encoded, shows, shown }: Points) {
175  let best: Shape | undefined
176  if (shown < POINTS) {
177    let bestMatches = -1
178    for (const candidate of SHAPES) {
179      let matches = 0
180      for (let i = 0; i < POINTS; i++) matches += candidate.inside[i] === shows[i] ? 1 : 0
181      if (matches > bestMatches) {
182        bestMatches = matches
183        best = candidate
184      }
185    }
186    return best
187  }
188  // Each split's error is the total of the squares less what its two groups' averages explain: the
189  // largest |sum|² / count over the two groups is the least error.
190  const total = [0, 0, 0]
191  for (let i = 0; i < POINTS; i++) for (let c = 0; c < 3; c++) total[c]! += encoded[i * 3 + c]!
192  let bestScore = (total[0]! ** 2 + total[1]! ** 2 + total[2]! ** 2) / POINTS
193  for (const candidate of SHAPES) {
194    const sum = [0, 0, 0]
195    let count = 0
196    for (let i = 0; i < POINTS; i++) {
197      if (candidate.inside[i] === 0) continue
198      for (let c = 0; c < 3; c++) sum[c]! += encoded[i * 3 + c]!
199      count++
200    }
201    let score = 0
202    for (let c = 0; c < 3; c++) score += sum[c]! ** 2 / count + (total[c]! - sum[c]!) ** 2 / (POINTS - count)
203    if (score > bestScore + 1e-6) {
204      bestScore = score
205      best = candidate
206    }
207  }
208  return best
209}
210
211/** The color of the points that show on one side of a shape (all of them with none), mixed in linear light. */
212function colorOf ({ linear, shows }: Points, shape: Shape | undefined, side: number) {
213  const sum = [0, 0, 0]
214  let count = 0
215  for (let i = 0; i < POINTS; i++) {
216    if (shows[i] === 0 || (shape ? shape.inside[i] : 1) !== side) continue
217    for (let c = 0; c < 3; c++) sum[c]! += linear[i * 3 + c]!
218    count++
219  }
220  return count === 0 ? DEFAULT_COLOR : (encode(sum[0]! / count) << 16) | (encode(sum[1]! / count) << 8) | encode(sum[2]! / count)
221}
222
223/**
224 * The picture as a `Raster`'s cells: `columns × rows` block elements, each cell sampled at eight
225 * by eight points and drawn with the half, eighth or quarter block that splits them best, in their
226 * two colors; the terminal's own color where the picture is see-through.
227 */
228export function toCells (pixels: Pixels, columns: number, rows: number): string {
229  const words = new Uint32Array(columns * rows * 3)
230  const points: Points = { encoded: new Float64Array(POINTS * 3), linear: new Float64Array(POINTS * 3), shows: new Uint8Array(POINTS), shown: 0 }
231  for (let row = 0; row < rows; row++) {
232    for (let column = 0; column < columns; column++) {
233      pointsOf(pixels, columns, rows, column, row, points)
234      const at = (row * columns + column) * 3
235      if (points.shown === 0) {
236        words.set([SPACE, DEFAULT_COLOR, DEFAULT_COLOR], at)
237        continue
238      }
239      const best = shapeOf(points)
240      const inside = colorOf(points, best, 1)
241      const outside = !best ? inside : points.shown < POINTS ? DEFAULT_COLOR : colorOf(points, best, 0)
242      words.set([best?.glyph ?? UPPER_HALF, inside, outside], at)
243    }
244  }
245  return new Uint8Array(words.buffer).toBase64()
246}
247

1// The programs that show a picture in a window of its own, by the pointer, outside the terminal:
2// one a system, each run with what it needs in the environment (IMAGE_PREVIEW_FILE, the picture;
3// IMAGE_PREVIEW_SIZE, its longest side at most; IMAGE_PREVIEW_LIFE, how many seconds at most it
4// stays). Each writes `shown` once the window is up, or a line starting `error`, then a `.` every
5// tenth of a second, so the one reading it is woken to end it, and it ends when nobody reads. The
6// window never takes the focus or the pointer: the terminal keeps both.
7
8/** macOS: JavaScript for Automation, run by `osascript`, drawing an AppKit panel. */
9export const MACOS = String.raw`
10ObjC.import('Cocoa')
11
12function run () {
13  const env = $.NSProcessInfo.processInfo.environment
14  const read = key => { const value = env.objectForKey(key); return value.isNil() ? '' : ObjC.unwrap(value) }
15  const say = text => $.NSFileHandle.fileHandleWithStandardOutput.writeData($(text + '\n').dataUsingEncoding($.NSUTF8StringEncoding))
16  const largest = Number(read('IMAGE_PREVIEW_SIZE')) || 800
17  // An accessory app: no Dock icon, never the active one.
18  $.NSApplication.sharedApplication.setActivationPolicy(1)
19  const image = $.NSImage.alloc.initWithContentsOfFile(read('IMAGE_PREVIEW_FILE'))
20  if (image.isNil() || image.representations.count === 0) { say('error: cannot read the image'); return }
21  const pixels = image.representations.firstObject
22  const mouse = $.NSEvent.mouseLocation
23  const screens = $.NSScreen.screens
24  let screen = $.NSScreen.mainScreen
25  for (let i = 0; i < screens.count; i++) {
26    const frame = screens.objectAtIndex(i).frame
27    if (mouse.x >= frame.origin.x && mouse.x < frame.origin.x + frame.size.width && mouse.y >= frame.origin.y && mouse.y < frame.origin.y + frame.size.height) screen = screens.objectAtIndex(i)
28  }
29  const area = screen.visibleFrame
30  let width = pixels.pixelsWide / screen.backingScaleFactor
31  let height = pixels.pixelsHigh / screen.backingScaleFactor
32  const fit = Math.min(1, largest / Math.max(width, height), (area.size.width * 0.7) / width, (area.size.height * 0.7) / height)
33  width = Math.round(width * fit)
34  height = Math.round(height * fit)
35  const pad = 6
36  const gap = 18
37  const outer = { width: width + pad * 2, height: height + pad * 2 }
38  const top = area.origin.y + area.size.height
39  const right = area.origin.x + area.size.width
40  // Above the pointer and to its right; under it, or to its left, where there is no room: never
41  // over the pointer, which stays on the terminal.
42  let x = mouse.x + gap
43  let y = mouse.y + gap
44  if (y + outer.height > top) y = mouse.y - gap - outer.height
45  if (x + outer.width > right) x = mouse.x - gap - outer.width
46  x = Math.max(area.origin.x, Math.min(x, right - outer.width))
47  y = Math.max(area.origin.y, Math.min(y, top - outer.height))
48  // Borderless and non-activating; over every app window, on every space, full-screen ones too.
49  const panel = $.NSPanel.alloc.initWithContentRectStyleMaskBackingDefer($.NSMakeRect(x, y, outer.width, outer.height), 128, 2, false)
50  panel.level = 25
51  panel.ignoresMouseEvents = true
52  panel.hasShadow = true
53  panel.opaque = false
54  panel.backgroundColor = $.NSColor.clearColor
55  panel.collectionBehavior = 1 | 16 | 256
56  const backdrop = $.NSVisualEffectView.alloc.initWithFrame($.NSMakeRect(0, 0, outer.width, outer.height))
57  backdrop.material = 13
58  backdrop.state = 1
59  backdrop.wantsLayer = true
60  backdrop.layer.cornerRadius = 10
61  backdrop.layer.masksToBounds = true
62  const view = $.NSImageView.alloc.initWithFrame($.NSMakeRect(pad, pad, width, height))
63  view.image = image
64  view.imageScaling = 3
65  backdrop.addSubview(view)
66  panel.contentView = backdrop
67  panel.orderFrontRegardless
68  say('shown')
69  const end = Date.now() + (Number(read('IMAGE_PREVIEW_LIFE')) || 120) * 1000
70  while (Date.now() < end) {
71    $.NSRunLoop.currentRunLoop.runUntilDate($.NSDate.dateWithTimeIntervalSinceNow(0.1))
72    say('.')
73  }
74}
75`
76
77/** Linux and other X11 desktops: Python's Tk, an undecorated tooltip window. The picture is a PNG. */
78export const LINUX = String.raw`
79import math, os, sys
80try:
81    import tkinter as tk
82except ImportError:
83    print('error: no tkinter', flush=True)
84    sys.exit(1)
85
86largest = int(os.environ.get('IMAGE_PREVIEW_SIZE') or 800)
87life = float(os.environ.get('IMAGE_PREVIEW_LIFE') or 120)
88try:
89    root = tk.Tk()
90except tk.TclError as error:
91    print('error: no display: %s' % error, flush=True)
92    sys.exit(1)
93root.withdraw()
94root.overrideredirect(True)
95for name, value in (('-topmost', True), ('-type', 'tooltip')):
96    try:
97        root.attributes(name, value)
98    except tk.TclError:
99        pass
100try:
101    image = tk.PhotoImage(file=os.environ.get('IMAGE_PREVIEW_FILE', ''))
102except tk.TclError:
103    print('error: cannot read the image', flush=True)
104    sys.exit(1)
105screen_width, screen_height = root.winfo_screenwidth(), root.winfo_screenheight()
106step = max(1, math.ceil(max(image.width() / min(largest, screen_width * 0.7), image.height() / min(largest, screen_height * 0.7))))
107if step > 1:
108    image = image.subsample(step)
109pad, gap = 6, 18
110width, height = image.width() + pad * 2, image.height() + pad * 2
111root.configure(background='#202020')
112tk.Label(root, image=image, borderwidth=0, highlightthickness=0, background='#202020').pack(padx=pad, pady=pad)
113mouse_x, mouse_y = root.winfo_pointerxy()
114# Above the pointer and to its right; under it, or to its left, where there is no room: never over
115# the pointer, which stays on the terminal.
116x, y = mouse_x + gap, mouse_y - gap - height
117if y < 0:
118    y = mouse_y + gap
119if x + width > screen_width:
120    x = mouse_x - gap - width
121x = max(0, min(x, screen_width - width))
122y = max(0, min(y, screen_height - height))
123root.geometry('%dx%d+%d+%d' % (width, height, x, y))
124root.deiconify()
125root.update_idletasks()
126print('shown', flush=True)
127
128
129def beat():
130    try:
131        print('.', flush=True)
132    except (BrokenPipeError, OSError):
133        # Nobody reads: end, the pipe's last flush going nowhere.
134        os.dup2(os.open(os.devnull, os.O_WRONLY), sys.stdout.fileno())
135        root.destroy()
136        return
137    root.after(100, beat)
138
139
140beat()
141root.after(int(life * 1000), root.destroy)
142root.mainloop()
143`
144
145/** Windows: PowerShell and Windows Forms, a tool window that is never activated. */
146export const WINDOWS = String.raw`
147$ErrorActionPreference = 'Stop'
148Add-Type -AssemblyName System.Windows.Forms, System.Drawing
149$largest = if ($env:IMAGE_PREVIEW_SIZE) { [int]$env:IMAGE_PREVIEW_SIZE } else { 800 }
150$life = if ($env:IMAGE_PREVIEW_LIFE) { [int]$env:IMAGE_PREVIEW_LIFE } else { 120 }
151# A form that shows without taking the focus, as a tool window, topmost, the pointer passing through.
152# Compiled once into the temp folder, then loaded from there: compiling takes a second.
153$assembly = Join-Path $env:TEMP 'image-preview-popup-1.dll'
154if (-not (Test-Path $assembly)) {
155  try {
156  Add-Type -OutputAssembly $assembly -OutputType Library -ReferencedAssemblies System.Windows.Forms, System.Drawing -TypeDefinition @'
157public class ImagePreviewPopup : System.Windows.Forms.Form {
158  protected override bool ShowWithoutActivation { get { return true; } }
159  protected override System.Windows.Forms.CreateParams CreateParams {
160    get {
161      System.Windows.Forms.CreateParams p = base.CreateParams;
162      p.ExStyle |= 0x08000000 | 0x00000080 | 0x00000008 | 0x00000020;
163      return p;
164    }
165  }
166}
167'@
168  } catch {
169    # Another preview compiling it at the same time: its file is used.
170  }
171}
172Add-Type -Path $assembly
173try { $image = [System.Drawing.Image]::FromFile($env:IMAGE_PREVIEW_FILE) } catch { [Console]::Out.WriteLine('error: cannot read the image'); exit 1 }
174$cursor = [System.Windows.Forms.Cursor]::Position
175$area = [System.Windows.Forms.Screen]::FromPoint($cursor).WorkingArea
176$fit = [Math]::Min(1.0, [Math]::Min($largest / [Math]::Max($image.Width, $image.Height), [Math]::Min($area.Width * 0.7 / $image.Width, $area.Height * 0.7 / $image.Height)))
177$pad = 6
178$gap = 18
179$width = [int]($image.Width * $fit) + $pad * 2
180$height = [int]($image.Height * $fit) + $pad * 2
181# Above the pointer and to its right; under it, or to its left, where there is no room: never over
182# the pointer, which stays on the terminal.
183$x = $cursor.X + $gap
184$y = $cursor.Y - $gap - $height
185if ($y -lt $area.Top) { $y = $cursor.Y + $gap }
186if ($x + $width -gt $area.Right) { $x = $cursor.X - $gap - $width }
187$x = [Math]::Max($area.Left, [Math]::Min($x, $area.Right - $width))
188$y = [Math]::Max($area.Top, [Math]::Min($y, $area.Bottom - $height))
189$form = New-Object ImagePreviewPopup
190$form.FormBorderStyle = 'None'
191$form.ShowInTaskbar = $false
192$form.TopMost = $true
193$form.StartPosition = 'Manual'
194$form.Bounds = New-Object System.Drawing.Rectangle($x, $y, $width, $height)
195$form.BackColor = [System.Drawing.Color]::FromArgb(32, 32, 32)
196$form.Padding = New-Object System.Windows.Forms.Padding($pad)
197$form.Opacity = 0.99
198$box = New-Object System.Windows.Forms.PictureBox
199$box.Dock = 'Fill'
200$box.SizeMode = 'Zoom'
201$box.Image = $image
202$form.Controls.Add($box)
203$form.add_Shown({ [Console]::Out.WriteLine('shown'); [Console]::Out.Flush() })
204$timer = New-Object System.Windows.Forms.Timer
205$timer.Interval = $life * 1000
206$timer.add_Tick({ $form.Close() })
207$timer.Start()
208$beat = New-Object System.Windows.Forms.Timer
209$beat.Interval = 100
210$beat.add_Tick({ try { [Console]::Out.WriteLine('.'); [Console]::Out.Flush() } catch { $form.Close() } })
211$beat.Start()
212[System.Windows.Forms.Application]::Run($form)
213`
214
215/**
216 * Windows: PowerShell and System.Drawing, the converter where ImageMagick is not installed. Writes
217 * IMAGE_PREVIEW_FILE shrunk to IMAGE_PREVIEW_SIZE at most, as IMAGE_PREVIEW_FORMAT (`bmp` or `png`),
218 * to IMAGE_PREVIEW_OUT.
219 */
220export const WINDOWS_CONVERT = String.raw`
221$ErrorActionPreference = 'Stop'
222Add-Type -AssemblyName System.Drawing
223$source = [System.Drawing.Image]::FromFile($env:IMAGE_PREVIEW_FILE)
224$largest = if ($env:IMAGE_PREVIEW_SIZE) { [int]$env:IMAGE_PREVIEW_SIZE } else { [Math]::Max($source.Width, $source.Height) }
225$fit = [Math]::Min(1.0, $largest / [Math]::Max($source.Width, $source.Height))
226$width = [Math]::Max(1, [int]($source.Width * $fit))
227$height = [Math]::Max(1, [int]($source.Height * $fit))
228$picture = New-Object System.Drawing.Bitmap($width, $height, [System.Drawing.Imaging.PixelFormat]::Format32bppArgb)
229$canvas = [System.Drawing.Graphics]::FromImage($picture)
230$canvas.InterpolationMode = [System.Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic
231$canvas.DrawImage($source, 0, 0, $width, $height)
232$canvas.Dispose()
233$format = if ($env:IMAGE_PREVIEW_FORMAT -eq 'png') { [System.Drawing.Imaging.ImageFormat]::Png } else { [System.Drawing.Imaging.ImageFormat]::Bmp }
234$picture.Save($env:IMAGE_PREVIEW_OUT, $format)
235`
236
237/** A PowerShell script as `-EncodedCommand` takes it: UTF-16LE, base64. */
238export function encodedCommand (script: string) {
239  const bytes = new Uint8Array(script.length * 2)
240  for (let i = 0; i < script.length; i++) {
241    const code = script.charCodeAt(i)
242    bytes[i * 2] = code & 0xff
243    bytes[i * 2 + 1] = code >> 8
244  }
245  return bytes.toBase64()
246}
247
248/** How PowerShell runs one of these scripts: no profile, no prompts, no script policy in the way. */
249export const powershell = (script: string) => ['powershell', '-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-EncodedCommand', encodedCommand(script)]
250

1// The surface module of the pointer targets: a chip that says when the pointer is on it, so the
2// hooks module shows that image's card, and hides it when the pointer leaves; and when it is
3// clicked, so the hooks module opens the image.
4import type { ClientModule } from 'claude-code'
5
6/** `label`, the chip's text; none (or no props), an unpainted region over a row the engine draws. */
7type Props = { label?: string } | null
8
9const REPEAT_MS = 50
10
11const Chip: ClientModule<Props, { isOver: boolean; isPressed?: boolean }> = (props, surface) => {
12  const label = props?.label
13  surface.onPointer(event => {
14    // A click is the left button let go over the chip it went down on.
15    if (event.type === 'down') {
16      if (event.button === 'left') surface.setState({ isOver: true, isPressed: true })
17      return
18    }
19    if (event.type === 'up') {
20      // A region not laid out yet (0 by 0) says nothing of where the pointer is.
21      const isInside = surface.columns === 0 || (event.x >= 0 && event.y >= 0 && event.x < surface.columns && event.y < surface.rows)
22      if (surface.state?.isPressed && isInside) surface.post({ isOver: true, isClicked: true })
23      surface.setState({ isOver: isInside })
24      return
25    }
26    // A move over the chip says so again: a card hidden while the pointer stayed comes back.
27    const isOver = event.type !== 'leave'
28    surface.post({ isOver })
29    if (isOver !== (surface.state?.isOver ?? false)) surface.setState({ isOver, isPressed: surface.state?.isPressed })
30    // Going straight from one chip to the next, the leave and the enter post in one frame, and the
31    // later replaces the other: the enter says so again a frame on.
32    if (event.type === 'enter') {
33      const stop = surface.every(REPEAT_MS, () => {
34        stop()
35        if (surface.state?.isOver) surface.post({ isOver: true })
36      })
37    }
38  })
39  const { Box, Text } = surface.elements
40  if (label === undefined) return <Box />
41  return <Text color="claude" inverse={surface.state?.isOver === true}>{label}</Text>
42}
43
44export default Chip
45

1/** A pasted image as the mod read it: its file, format, size and pixel shape, or why it has no preview. */
2export type Picture =
3  | { file: string; format: string; bytes: number; width: number; height: number; png?: string }
4  | { error: string }
5
6/**
7 * The card on show: its image; the pointer target that showed it (`<site>/<chip key>`); the
8 * transcript row it floats on, by `requestId` (null: the band holds it, or a window); how many rows
9 * it may take; and whether it stands on that row's last row on screen, ends just above its first,
10 * or is a window of its own, outside the terminal.
11 */
12export type Shown = { id: number; target: string; host: string | null; rows: number; edge: 'last' | 'first' | 'window' }
13
14declare module 'claude-code' {
15  interface PluginState {
16    'image-preview': {
17      /** The `[Image #N]` ids in the prompt box, in the order typed. */
18      draft: number[]
19      /** Each image read so far, by id. */
20      pictures: Record<string, Picture>
21      /** The card the pointer has open, and where it floats; null while none is. */
22      shown: Shown | null
23      /** Whether a transcript row, by its `requestId`, holds the card: so only the rows it leaves and joins draw again. */
24      hosting: StateFamily<boolean>
25    }
26  }
27}
28