inline-images

1import type { ElementTable, EngineInterface, Register, RenderElement } from 'claude-code'
2
3/** An image to show: its own file, the small PNG drawn for it, its size in pixels. */
4type Picture = { path: string; thumb: string; width: number; height: number }
5type Shown = { label: string; picture: Picture }
6type Cells = { columns: number; rows: number }
7
8const MARKER = /\[Image #(\d+)\]/g
9const IMAGE_PATH = /\/[^\s'"`<>|;]+\.(?:png|jpe?g|gif|webp|heic|tiff?|bmp)\b/gi
10const CMUX = '/Applications/cmux.app/Contents/Resources/bin/cmux'
11const CACHE = '/private/tmp/inline-images-cache'
12
13// Terminal cells are about twice as tall as they are wide, and ~8 px wide.
14const CELL_ASPECT = 0.5
15const CELL_PX = 8
16// Thumbnails are drawn from a copy at most this many pixels on the long side,
17// so the terminal never scales a full-size screenshot on a redraw.
18const THUMB_PX = 480
19// Size caps for a picture, in cells: alone in a row, several side by side, and
20// the preview above the prompt box. The frame adds a cell on each side.
21const SINGLE: Cells = { columns: 32, rows: 8 }
22const MULTI: Cells = { columns: 22, rows: 6 }
23const PREVIEW: Cells = { columns: 18, rows: 5 }
24const GAP = 2
25// The narrowest caption: room for a size and the Open button.
26const MIN_TILE = 18
27const MAX_IMAGES = 6
28const POLL_MS = 500
29const SEARCH_MS = 5000
30
31// Pictures by image path, or null when the file is gone or cannot be drawn.
32// Module state: a reload of the mod starts it over.
33const pictures = new Map<string, Picture | null>()
34const loading = new Set<string>()
35// This session's paste folder, the image numbers in the prompt box, and the
36// highest number already sent (-1 until the first check).
37let pasteFolder = ''
38let draft: number[] = []
39let maxSent = -1
40let lastSearch = 0
41// The first tool row that named each image file: later rows naming it again
42// (a screenshot, then a Read of it) show nothing more.
43const firstRow = new Map<string, string>()
44
45export const register: Register = on => {
46  // Pasted images not sent yet show above the prompt box. Claude Code saves a
47  // paste at once, and mods do not hear prompt-box edits in every session, so
48  // the paste folder is checked on a timer.
49  on('session.start', async ($, e, next) => {
50    const result = await next(e)
51    $.clock.every(POLL_MS, () => void poll($))
52    return result
53  })
54
55  on('prompt.submit', async ($, e, next) => {
56    const result = await next(e)
57    markSent($, markersOf(e.text))
58    return result
59  })
60
61  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
62    const folder = pasteFolder
63    const shown = folder ? draft.flatMap(n => shownOf(`${folder}/${n}.png`, `#${n}`)) : []
64    if (e.surface !== 'terminal' || e.props.hasSurvey || shown.length === 0) return next(e)
65    const els = $.ui.resolve(e)
66    const limit = { columns: PREVIEW.columns, rows: Math.max(2, Math.min(PREVIEW.rows, e.props.maxRows - 3)) }
67    return (
68      <els.Box flexDirection="row" columnGap={GAP} paddingLeft={2}>
69        {shown.map((item, i) => (
70          <els.Box key={`draft-${i}`} flexDirection="column">
71            {framed(els, item, cellsOf(item.picture, limit), `draft-${i}`)}
72            <els.Text dimColor>{item.label}</els.Text>
73          </els.Box>
74        ))}
75      </els.Box>
76    )
77  })
78
79  // Your prompt rows: the pasted pictures under the text.
80  on('ui.render', { component: 'UserMessage' }, async ($, e, next) => {
81    const row = await next(e)
82    if (e.surface !== 'terminal') return row
83    const markers = markersOf(e.props.text)
84    if (markers.length === 0) return row
85    markSent($, markers)
86    const folder = pasteFolder || (await pasteFolderOf($))
87    if (!folder) return row
88    const paths = markers.map(n => `${folder}/${n}.png`)
89    request($, paths)
90    const shown = markers.flatMap(n => shownOf(`${folder}/${n}.png`, `#${n}`))
91    return gallery($.ui.resolve(e), row, shown, e.viewport?.columns, actionsOf($))
92  })
93
94  // A tool call that named an image file (Claude read it, wrote it, or took a
95  // screenshot into it): the pictures under its row.
96  on('ui.render', { component: 'ToolUse' }, async ($, e, next) => {
97    const row = await next(e)
98    if (e.surface !== 'terminal' || e.props.isRunning) return row
99    const shown = toolImages($, [e.props.input], e.requestId)
100    return gallery($.ui.resolve(e), row, shown, e.viewport?.columns, actionsOf($))
101  })
102
103  // A collapsed group (`Read 3 files`): the pictures of all its calls under its line.
104  on('ui.render', { component: 'ToolGroup' }, async ($, e, next) => {
105    const row = await next(e)
106    if (e.surface !== 'terminal' || e.props.isExpanded || e.props.isActive) return row
107    const shown = toolImages($, e.props.calls.map(call => call.input), e.requestId)
108    return gallery($.ui.resolve(e), row, shown, e.viewport?.columns, actionsOf($))
109  })
110
111  // A click on a picture opens it in a cmux preview tab.
112  on('ui.message', async ($, e) => {
113    const path = (e.data as { path?: unknown } | undefined)?.path
114    if (typeof e.element === 'string' && e.element.startsWith('hit-') && typeof path === 'string') {
115      void openFile($, path, true)
116    }
117    return {}
118  })
119}
120
121/** What a picture's buttons do, made in each hook from its own `$`. */
122type Actions = { open: (path: string) => void }
123
124function actionsOf($: EngineInterface): Actions {
125  return { open: path => void openFile($, path, false) }
126}
127
128/** A picture in a thin dim frame, with a clear click layer laid over it. */
129function framed(els: ElementTable<'terminal'>, item: Shown, box: Cells, key: string): RenderElement {
130  const { Box, Client, Image } = els
131  return (
132    <Box key={`frame-${key}`} borderStyle="round" borderDimColor width={box.columns + 2} height={box.rows + 2}>
133      <Image source={{ file: item.picture.thumb, format: 'png' }} alt={`[${item.label}]`} {...box} />
134      <Box position="absolute" top={0} left={0}>
135        <Client key={`hit-${key}`} module="./click.ts" width={box.columns} height={box.rows} props={{ path: item.picture.path }} />
136      </Box>
137    </Box>
138  )
139}
140
141/**
142 * The row with its pictures under it, side by side in tiles of one width:
143 * each picture framed, its name, then its size and an Open button.
144 */
145function gallery(
146  els: ElementTable<'terminal'>,
147  row: RenderElement,
148  shown: Shown[],
149  viewportColumns: number | undefined,
150  actions: Actions,
151): RenderElement {
152  if (shown.length === 0) return row
153  const { Box, Button, Text } = els
154  const width = (viewportColumns ?? 100) - 4
155  const visible = shown.slice(0, MAX_IMAGES)
156  const isSingle = visible.length === 1
157  const limit = isSingle ? SINGLE : MULTI
158  const fit = { columns: Math.min(limit.columns, width - 2), rows: limit.rows }
159  return (
160    <Box flexDirection="column">
161      {row}
162      <Box flexDirection="row" flexWrap="wrap" columnGap={GAP} rowGap={1} marginLeft={2} marginTop={1}>
163        {visible.map((item, i) => {
164          const box = cellsOf(item.picture, fit)
165          const tile = Math.max(MIN_TILE, (isSingle ? box.columns : fit.columns) + 2)
166          return (
167            <Box key={`image-${i}`} flexDirection="column" width={tile}>
168              <Box width={tile} height={(isSingle ? box.rows : fit.rows) + 2} alignItems="flex-end">
169                {framed(els, item, box, `image-${i}`)}
170              </Box>
171              <Text dimColor wrap="truncate-middle">{item.label}</Text>
172              <Box columnGap={2}>
173                <Text dimColor>{`${item.picture.width}×${item.picture.height}`}</Text>
174                <Button key={`open-${i}`} label="Open" plain dimColor onPress={() => actions.open(item.picture.path)} />
175              </Box>
176            </Box>
177          )
178        })}
179        {shown.length > visible.length && <Text dimColor>{`+${shown.length - visible.length}`}</Text>}
180      </Box>
181    </Box>
182  )
183}
184
185function shownOf(path: string, label: string): Shown[] {
186  const picture = pictures.get(path)
187  return picture ? [{ label, picture }] : []
188}
189
190/** The image files named anywhere in these tool inputs, as far as they are loaded. */
191function toolImages($: EngineInterface, inputs: unknown[], requestId: string): Shown[] {
192  const named = [...new Set(inputs.flatMap(input => stringsIn(input).flatMap(text => text.match(IMAGE_PATH) ?? [])))]
193  for (const path of named) if (!firstRow.has(path)) firstRow.set(path, requestId)
194  const paths = named.filter(path => firstRow.get(path) === requestId)
195  request($, paths)
196  return paths.flatMap(path => shownOf(path, path.slice(path.lastIndexOf('/') + 1)))
197}
198
199function stringsIn(value: unknown): string[] {
200  if (typeof value === 'string') return [value]
201  if (Array.isArray(value)) return value.flatMap(stringsIn)
202  if (value && typeof value === 'object') return Object.values(value).flatMap(stringsIn)
203  return []
204}
205
206function markersOf(text: string): number[] {
207  return [...new Set([...text.matchAll(MARKER)].map(match => Number(match[1])))]
208}
209
210/** Loads the pictures not known yet, then redraws. */
211function request($: EngineInterface, paths: string[]): void {
212  const missing = paths.filter(path => !pictures.has(path) && !loading.has(path))
213  if (missing.length > 0) void loadPictures($, missing)
214}
215
216async function loadPictures($: EngineInterface, paths: string[]): Promise<void> {
217  let isChanged = false
218  for (const path of paths) {
219    if (pictures.has(path) || loading.has(path)) continue
220    loading.add(path)
221    try {
222      const picture = await pictureOf($, path)
223      // A paste not written yet is looked for again on the next draw.
224      if (picture || !path.includes('/images/')) pictures.set(path, picture)
225      isChanged = isChanged || picture !== null
226    } catch (error) {
227      pictures.set(path, null)
228      $.ui.log(`inline-images: ${String(error)}`, { to: 'debug' })
229    } finally {
230      loading.delete(path)
231    }
232  }
233  if (isChanged) $.ui.invalidate('ui.render')
234}
235
236/** The image's size and a small PNG copy to draw, made once with macOS `sips`. */
237async function pictureOf($: EngineInterface, path: string): Promise<Picture | null> {
238  const stat = await $.fs.stat(path).catch(() => undefined)
239  if (stat?.kind !== 'file' || stat.size === 0) return null
240  const size = await sizeOf($, path)
241  if (!size) return null
242  const isSmallPng = /\.png$/i.test(path) && Math.max(size.width, size.height) <= THUMB_PX
243  if (isSmallPng) return { path, thumb: path, ...size }
244  const thumb = `${CACHE}/${hashOf(`${path}:${stat.mtimeMs}`)}.png`
245  if (!(await $.fs.exists(thumb))) {
246    await $.process.run(['/bin/mkdir', '-p', CACHE])
247    const made = await $.process.run(['/usr/bin/sips', '-s', 'format', 'png', '-Z', String(THUMB_PX), path, '--out', thumb])
248    if (made.exitCode !== 0) return null
249  }
250  return { path, thumb, ...size }
251}
252
253async function sizeOf($: EngineInterface, file: string): Promise<{ width: number; height: number } | undefined> {
254  const { exitCode, stdout } = await $.process.run(['/usr/bin/sips', '-g', 'pixelWidth', '-g', 'pixelHeight', file])
255  const width = Number(/pixelWidth: (\d+)/.exec(stdout)?.[1])
256  const height = Number(/pixelHeight: (\d+)/.exec(stdout)?.[1])
257  return exitCode === 0 && width > 0 && height > 0 ? { width, height } : undefined
258}
259
260function hashOf(text: string): string {
261  let hash = 0x811c9dc5
262  for (let i = 0; i < text.length; i++) {
263    hash ^= text.charCodeAt(i)
264    hash = Math.imul(hash, 0x01000193)
265  }
266  return (hash >>> 0).toString(16)
267}
268
269/**
270 * Opens an image. In cmux, a click puts it in the split beside Claude: the
271 * workspace's other pane, or a new browser split on the right when there is
272 * none (later clicks reuse it). The Open button, and anywhere outside cmux,
273 * hand it to the Mac's default app.
274 */
275async function openFile($: EngineInterface, path: string, inCmux: boolean): Promise<void> {
276  try {
277    const workspace = inCmux ? await $.env.get('CMUX_WORKSPACE_ID') : undefined
278    const surface = inCmux ? await $.env.get('CMUX_SURFACE_ID') : undefined
279    const hasCmux = workspace !== undefined && surface !== undefined && (await $.fs.exists(CMUX))
280    if (!hasCmux) {
281      const { exitCode, stderr } = await $.process.run(['/usr/bin/open', path])
282      if (exitCode !== 0) $.ui.toast(`Could not open ${path.slice(path.lastIndexOf('/') + 1)}: ${stderr.trim()}`)
283      return
284    }
285    const pane = await otherPane($, workspace, surface)
286    const { exitCode, stderr } = pane
287      ? await $.process.run([CMUX, 'open', path, '--workspace', workspace, '--pane', pane, '--focus', 'false'])
288      : await $.process.run([CMUX, 'new-pane', '--type', 'browser', '--direction', 'right', '--workspace', workspace, '--url', `file://${path}`, '--focus', 'false'])
289    if (exitCode !== 0) $.ui.toast(`Could not open ${path.slice(path.lastIndexOf('/') + 1)}: ${stderr.trim()}`)
290  } catch (error) {
291    $.ui.log(`inline-images: ${String(error)}`, { to: 'debug' })
292  }
293}
294
295/** A pane of the workspace other than the one Claude runs in, docks left out. */
296async function otherPane($: EngineInterface, workspace: string, surface: string): Promise<string | undefined> {
297  const listed = await $.process.run([CMUX, 'list-panes', '--workspace', workspace])
298  const panes = listed.stdout.split('\n').flatMap(line => {
299    const ref = /pane:\d+/.exec(line)?.[0]
300    return ref && !line.includes('[dock') ? [ref] : []
301  })
302  for (const pane of panes) {
303    const surfaces = await $.process.run([CMUX, 'list-pane-surfaces', '--workspace', workspace, '--pane', pane, '--id-format', 'both'])
304    if (!surfaces.stdout.includes(surface)) return pane
305  }
306  return undefined
307}
308
309function markSent($: EngineInterface, markers: number[]): void {
310  const highest = Math.max(maxSent, ...markers)
311  if (highest <= maxSent) return
312  maxSent = highest
313  draft = draft.filter(n => n > maxSent)
314  $.ui.invalidate('ui.render')
315}
316
317/** Shows the pasted images newer than the last one sent. */
318async function poll($: EngineInterface): Promise<void> {
319  try {
320    if (!pasteFolder) {
321      const now = await $.clock.now()
322      if (now - lastSearch < SEARCH_MS) return
323      lastSearch = now
324    }
325    const folder = await pasteFolderOf($)
326    if (!folder) {
327      if (maxSent < 0) maxSent = 0 // no folder yet: nothing pasted in this session
328      return
329    }
330    const numbers = (await $.fs.list(folder)).flatMap(entry => {
331      const match = /^(\d+)\.png$/.exec(entry.name)
332      return match ? [Number(match[1])] : []
333    })
334    if (maxSent < 0) {
335      maxSent = Math.max(0, ...numbers)
336      return
337    }
338    const fresh = numbers.filter(n => n > maxSent).sort((a, b) => a - b)
339    if (fresh.join() === draft.join()) return
340    draft = fresh
341    await loadPictures($, fresh.map(n => `${folder}/${n}.png`))
342    $.ui.invalidate('ui.render')
343  } catch (error) {
344    $.ui.log(`inline-images: ${String(error)}`, { to: 'debug' })
345  }
346}
347
348/** Claude Code keeps pastes in <tmp>/claude-<uid>/<project>/<session>/images. */
349async function pasteFolderOf($: EngineInterface): Promise<string> {
350  if (pasteFolder) return pasteFolder
351  const session = await $.session.id()
352  const uid = (await $.process.run(['/usr/bin/id', '-u'])).stdout.trim()
353  const userTmp = (await $.process.run(['/usr/bin/getconf', 'DARWIN_USER_TEMP_DIR'])).stdout.trim().replace(/\/+$/, '')
354  for (const base of [`/private/tmp/claude-${uid}`, `${userTmp}/claude-${uid}`]) {
355    for (const entry of await $.fs.list(base).catch(() => [])) {
356      const candidate = `${base}/${entry.name}/${session}/images`
357      if (entry.kind === 'dir' && (await $.fs.exists(candidate))) {
358        pasteFolder = candidate
359        return pasteFolder
360      }
361    }
362  }
363  return ''
364}
365
366/** The largest box of cells that keeps the picture's shape within the limits. */
367function cellsOf(picture: Picture, limit: Cells): Cells {
368  const rowsPerColumn = (picture.height / picture.width) * CELL_ASPECT
369  let columns = Math.max(4, Math.min(limit.columns, Math.ceil(picture.width / CELL_PX)))
370  let rows = Math.max(1, Math.round(columns * rowsPerColumn))
371  if (rows > limit.rows) {
372    rows = limit.rows
373    columns = Math.max(1, Math.round(rows / rowsPerColumn))
374  }
375  return { columns: Math.min(columns, 255), rows: Math.min(rows, 255) }
376}
377

1import type { ClientModule } from 'claude-code'
2
3/** A clear layer over a picture: a click on it posts the picture's path. */
4const ClickLayer: ClientModule<{ path: string }, undefined> = (props, surface) => {
5  surface.onPointer(event => {
6    if (event.type === 'up') surface.post({ path: props.path })
7  })
8  return surface.elements.Box({})
9}
10
11export default ClickLayer
12