image-preview

1/**
2 * image-preview: every pasted image becomes a chip that opens a preview.
3 *
4 * Chips sit under each prompt in the transcript that carried images, and in
5 * the band above the prompt for the `[Image #N]` placeholders in the draft.
6 * A click opens a small card under the chip, scaled to keep the picture's
7 * shape; the card's ↗ opens the whole picture in the system's own viewer.
8 *
9 * Pixels come from scripts/image.py (Pillow, run by uv), which decodes and
10 * scales an image. A draft's image is the file Claude Code cached for that
11 * paste, `<claude temp>/<project>/<session>/images/<n>.<ext>`. A sent
12 * prompt's images come whole from `session.append`, their paste numbers from
13 * the `imagePasteIds` its transcript line keeps; once those are known, the
14 * cached files stand in for the bytes.
15 *
16 * Everything that takes `$` is a top-level declaration: the loader inventories
17 * what a hooks module reaches for through `$`.
18 */
19import { atom, memberOf, read, update } from 'claude-code'
20import type { Args, EngineInterface, Register, RenderComponent, RenderElement, Timer } from 'claude-code'
21
22import type { DecodeSource, ImageData, ImageRef } from '../types'
23import { fitCells, parseDecoded, rasterCells } from './pixels'
24import type { CellSize, Decoded } from './pixels'
25import { imageNumbers, placeholders } from './placeholders'
26
27const ACCENT = 'suggestion'
28/**
29 * The card's picture at most; it shrinks to keep the shape, and to fit. Its
30 * 1024 cells are no more color pairs than a Raster paints as given.
31 */
32const HINT_ROOM: CellSize = { columns: 64, rows: 16 }
33/** A card's border, padding and indent around its picture. */
34const CARD_CHROME = 6
35/** The card's header needs this much even beside a narrow picture. */
36const CARD_MIN_COLUMNS = 24
37/** The decoder scales a picture down to fit this many pixels a side. */
38const DECODE_MAX = 400
39/** Decoded pictures held at most. */
40const DECODED_KEPT = 8
41/** Pasting an image raises no `prompt.edit`, so the draft is read on a timer. */
42const POLL_MS = 200
43/** A sent prompt's transcript line is written a moment after its row: look again this often, this many times. */
44const RELABEL_MS = 500
45const RELABEL_TRIES = 10
46/** Claude Code's temp folder under a temp root: `claude` on Windows, `claude-<uid>` elsewhere. */
47const CLAUDE_TEMP = /^claude(-\d+)?$/
48/** A file of the paste cache: `<paste number>.<ext>`. */
49const PASTED_FILE = /^(\d+)\.[A-Za-z0-9]+$/
50/**
51 * The helpers declare their own dependencies, Pillow for the decoder and none
52 * for the paste numbers; `--no-project` keeps them clear of the session's.
53 */
54const UV = ['uv', 'run', '--quiet', '--no-project', '--script']
55
56const MESSAGE_IMAGES = { plugin: 'image-preview', key: 'messageImages' } as const
57const IMAGE_SOURCE = { plugin: 'image-preview', key: 'imageSource' } as const
58const draftImages = atom({ plugin: 'image-preview', key: 'draftImages' } as const, [])
59const hint = atom({ plugin: 'image-preview', key: 'hint' } as const, null)
60
61/**
62 * Decoded pixels by image id, the last drawn last; past DECODED_KEPT the
63 * oldest goes. A reload starts it over; the next draw decodes again.
64 */
65const decoded = new Map<string, Promise<Decoded>>()
66/** Ids whose decode failed: the next click on the chip tries again. */
67const failed = new Set<string>()
68/** The cells worked out for a picture, by size: a redraw at a size already drawn reuses them. */
69const drawnCells = new WeakMap<Decoded, Map<string, string>>()
70/** Sizes kept for one picture: the card's, as the terminal is resized. */
71const SIZES_KEPT = 4
72/** The draft's image numbers as last written; unset after a reload, so the first read writes. */
73let draftNumbers: string | undefined
74let isReadingDraft = false
75/** The timer that reads the draft: one per module, whatever starts it again. */
76let draftPoll: Timer | undefined
77/** Where this session's paste cache and transcript are, found once per session id. */
78let pasteCache: { sessionId: string; dir: string } | undefined
79let transcript: { sessionId: string; path: string } | undefined
80
81/** A drawing that holds chips; its `requestId` says which chip's card is open. */
82type Site = { surface: 'terminal'; component: RenderComponent; requestId: string }
83type Preview = { decoded: Decoded } | { problem: string }
84type ContentBlock = Args<'session.append'>['message']['content'][number]
85
86const draftId = (number: number) => `draft:${number}`
87const draftNumber = (imageId: string) => (imageId.startsWith('draft:') ? Number(imageId.slice(6)) : undefined)
88const helperPath = ($: EngineInterface, script: string) => `${$.plugin.root}/scripts/${script}`
89const firstLine = (text: string) => text.trim().split(/\r?\n/)[0] ?? ''
90
91function cardRoom(columns: number, rows: number): CellSize {
92  return {
93    columns: Math.max(1, Math.min(HINT_ROOM.columns, columns - CARD_CHROME)),
94    rows: Math.max(1, Math.min(HINT_ROOM.rows, rows)),
95  }
96}
97
98function caption(image: ImageRef, preview: Preview): string {
99  if (!('decoded' in preview)) return image.label
100  return `${image.label} · ${preview.decoded.originalWidth}×${preview.decoded.originalHeight}`
101}
102
103/** An image block's bytes, when the block is one the engine holds inline. */
104function imageData(block: ContentBlock): ImageData[] {
105  const source = block.source as { type?: unknown; media_type?: unknown; data?: unknown } | undefined
106  if (block.type !== 'image' || source?.type !== 'base64' || typeof source.data !== 'string') return []
107  const mediaType = typeof source.media_type === 'string' ? source.media_type : 'image'
108  return [{ mediaType, base64: source.data }]
109}
110
111function textOf(block: ContentBlock): string {
112  return block.type === 'text' && typeof block.text === 'string' ? block.text : ''
113}
114
115// ── Claude Code's files ─────────────────────────────────────────────────────
116
117/**
118 * Where Claude Code's temp folder may be: `claude` on Windows, `claude-<uid>`
119 * on macOS and Linux, under `CLAUDE_CODE_TMPDIR` when it is set, else under
120 * the system's temp folder. Each is found by its name, since the session
121 * folder under it tells which one is this person's.
122 */
123async function claudeTemps($: EngineInterface): Promise<string[]> {
124  const custom = await $.env.get('CLAUDE_CODE_TMPDIR')
125  const system = [await $.env.get('TEMP'), await $.env.get('TMP'), await $.env.get('TMPDIR'), '/tmp']
126  const bases = (custom ? [custom] : system).flatMap((base) => (base ? [base.replace(/[\\/]+$/, '')] : []))
127
128  const temps: string[] = []
129  for (const base of new Set(bases)) {
130    for (const entry of await $.fs.list(base).catch(() => [])) {
131      if (entry.kind === 'dir' && CLAUDE_TEMP.test(entry.name)) temps.push(`${base}/${entry.name}`)
132    }
133  }
134  return temps
135}
136
137/**
138 * `tail` under the one project folder of `root` that holds it. A project
139 * folder is named after a working directory that may have moved since, so
140 * this session's is found by what it holds instead of rebuilt.
141 */
142async function underProject($: EngineInterface, root: string, tail: string): Promise<string | undefined> {
143  for (const entry of await $.fs.list(root).catch(() => [])) {
144    const path = `${root}/${entry.name}/${tail}`
145    if (entry.kind === 'dir' && (await $.fs.exists(path))) return path
146  }
147  return undefined
148}
149
150async function pasteCacheDir($: EngineInterface): Promise<string | undefined> {
151  const sessionId = await $.session.id()
152  if (pasteCache?.sessionId === sessionId) return pasteCache.dir
153  for (const root of await claudeTemps($)) {
154    const dir = await underProject($, root, `${sessionId}/images`)
155    if (dir !== undefined) {
156      pasteCache = { sessionId, dir }
157      return dir
158    }
159  }
160  return undefined
161}
162
163/** The files Claude Code cached for this session's pastes, by paste number, whatever their extension. */
164async function pastedFiles($: EngineInterface): Promise<Map<number, string>> {
165  const files = new Map<number, string>()
166  const dir = await pasteCacheDir($)
167  if (dir === undefined) return files
168  for (const entry of await $.fs.list(dir).catch(() => [])) {
169    const paste = entry.kind === 'file' ? PASTED_FILE.exec(entry.name) : null
170    if (paste) files.set(Number(paste[1]), `${dir}/${entry.name}`)
171  }
172  return files
173}
174
175/** The transcript a classic event names: each prompt, and each session that starts. */
176function noteTranscript(e: { session_id: string; transcript_path: string }): void {
177  if (e.transcript_path) transcript = { sessionId: e.session_id, path: e.transcript_path }
178}
179
180/**
181 * The session's transcript, as a classic event named it; after a reload,
182 * until the next prompt names it, found by its name under the config folder.
183 */
184async function transcriptPath($: EngineInterface): Promise<string | undefined> {
185  const sessionId = await $.session.id()
186  if (transcript?.sessionId === sessionId) return transcript.path
187  const home = (await $.env.get('USERPROFILE')) ?? (await $.env.get('HOME'))
188  const config = (await $.env.get('CLAUDE_CONFIG_DIR')) ?? (home ? `${home}/.claude` : undefined)
189  const path = config === undefined ? undefined : await underProject($, `${config}/projects`, `${sessionId}.jsonl`)
190  if (path !== undefined) transcript = { sessionId, path }
191  return path
192}
193
194// ── Pixels ──────────────────────────────────────────────────────────────────
195
196async function runDecoder($: EngineInterface, source: DecodeSource): Promise<Decoded> {
197  // The first run also has uv fetch Pillow, so it is given the longer wait.
198  const argv = [...UV, helperPath($, 'image.py'), 'decode', '--max', String(DECODE_MAX)]
199  const result =
200    'path' in source
201      ? await $.process.run([...argv, '--path', source.path], { timeoutMs: 120_000 })
202      : await $.process.run(argv, { stdin: source.base64, timeoutMs: 120_000 })
203  if (result.exitCode !== 0) {
204    const what = 'path' in source ? source.path.split('/').pop() : source.mediaType
205    throw new Error(`Cannot decode ${what}: ${firstLine(result.stderr)}`)
206  }
207  return parseDecoded(result.stdout)
208}
209
210/** Where an image's bytes are: the paste cache for a draft's, state for a sent one's. */
211async function sourceOf($: EngineInterface, imageId: string): Promise<DecodeSource> {
212  const number = draftNumber(imageId)
213  if (number !== undefined) {
214    const path = (await pastedFiles($)).get(number)
215    if (path === undefined) throw new Error(`Claude Code holds no pasted image #${number} in this session.`)
216    return { path }
217  }
218  const { value: source } = await $.state.get({ ...IMAGE_SOURCE, id: imageId })
219  if (!source) throw new Error('This image is no longer held.')
220  return source
221}
222
223function forgetDecoded(imageId: string): void {
224  decoded.delete(imageId)
225  failed.delete(imageId)
226}
227
228/**
229 * The image's pixels, decoded once. The promise is kept before anything is
230 * awaited, so a second drawing of the same image waits on the first decode.
231 */
232function decodeImage($: EngineInterface, imageId: string): Promise<Decoded> {
233  const known = decoded.get(imageId)
234  const pending = known ?? sourceOf($, imageId).then((source) => runDecoder($, source))
235  decoded.delete(imageId)
236  decoded.set(imageId, pending)
237  if (!known) pending.catch(() => failed.add(imageId))
238
239  for (const oldest of decoded.keys()) {
240    if (decoded.size <= DECODED_KEPT) break
241    forgetDecoded(oldest)
242  }
243  return pending
244}
245
246/**
247 * A draft's `[Image #N]` names a paste of the session it is in: once another
248 * session starts, the pixels decoded for it are another session's picture.
249 */
250function forgetDrafts(): void {
251  for (const imageId of decoded.keys()) {
252    if (draftNumber(imageId) !== undefined) forgetDecoded(imageId)
253  }
254  // Unset, so the next tick writes the chips for the new session.
255  draftNumbers = undefined
256}
257
258/** Raster cells for the picture at `size`, worked out once per size. */
259function cellsFor(picture: Decoded, size: CellSize): string {
260  const key = `${size.columns}x${size.rows}`
261  const sizes = drawnCells.get(picture) ?? new Map<string, string>()
262  drawnCells.set(picture, sizes)
263
264  const cells = sizes.get(key) ?? rasterCells(picture, size)
265  sizes.delete(key)
266  sizes.set(key, cells)
267  for (const oldest of sizes.keys()) {
268    if (sizes.size <= SIZES_KEPT) break
269    sizes.delete(oldest)
270  }
271  return cells
272}
273
274async function loadPreview($: EngineInterface, imageId: string): Promise<Preview> {
275  try {
276    return { decoded: await decodeImage($, imageId) }
277  } catch (error) {
278    return { problem: error instanceof Error ? error.message : String(error) }
279  }
280}
281
282// ── Sent prompts ────────────────────────────────────────────────────────────
283
284async function rememberImages($: EngineInterface, uuid: string, content: readonly ContentBlock[]): Promise<void> {
285  const pictures = content.flatMap(imageData)
286  if (pictures.length === 0) return
287
288  // Until the transcript line says which pastes these are, the tags in the
289  // text name them when they match the images one to one.
290  const numbers = imageNumbers(content.map(textOf).join('\n'))
291  const images = pictures.map((_, index) => ({
292    id: `${uuid}:${index}`,
293    label: numbers.length === pictures.length ? `Image #${numbers[index]}` : `Image ${index + 1}`,
294  }))
295
296  // Held until the transcript line names the pastes; `keepPastes` lets them go.
297  for (const [index, image] of images.entries()) {
298    const picture = pictures[index]
299    if (picture) await $.state.set({ ...IMAGE_SOURCE, id: image.id }, picture)
300  }
301  await $.state.set({ ...MESSAGE_IMAGES, id: uuid }, images)
302  $.clock.after(RELABEL_MS, () => void relabel($, uuid, RELABEL_TRIES))
303}
304
305/** The paste numbers row `uuid`'s transcript line keeps; null until the line is written. */
306async function pasteIds($: EngineInterface, uuid: string): Promise<number[] | null> {
307  const path = await transcriptPath($)
308  if (path === undefined) return null
309  const argv = [...UV, helperPath($, 'paste_ids.py'), '--path', path, '--uuid', uuid]
310  const result = await $.process.run(argv, { timeoutMs: 30_000 })
311  if (result.exitCode === 4) return []
312  if (result.exitCode !== 0) return null
313  return result.stdout.trim().split(',').filter(Boolean).map(Number)
314}
315
316/**
317 * Names each image by the paste it came from: `imagePasteIds` keeps one
318 * number per image block, in order. A typed `[Image #1]` reads the same as a
319 * pasted one, so the tags in the text alone can misname them.
320 */
321async function relabel($: EngineInterface, uuid: string, tries: number): Promise<void> {
322  try {
323    const ids = await pasteIds($, uuid)
324    if (ids === null) {
325      if (tries > 1) $.clock.after(RELABEL_MS, () => void relabel($, uuid, tries - 1))
326      return
327    }
328    const { value: images } = await $.state.get({ ...MESSAGE_IMAGES, id: uuid })
329    if (!images || ids.length !== images.length) return
330    const labeled = images.map((image, index) => ({ ...image, label: `Image #${ids[index]}` }))
331    await $.state.set({ ...MESSAGE_IMAGES, id: uuid }, labeled)
332    await keepPastes($, images, ids)
333  } catch (error) {
334    $.ui.log(`image-preview: reading paste numbers failed: ${String(error)}`, { to: 'debug' })
335  }
336}
337
338/**
339 * Swaps each image's bytes for the file Claude Code cached for its paste: the
340 * session then holds a path, not megabytes. Bytes with no file stay.
341 */
342async function keepPastes($: EngineInterface, images: ImageRef[], ids: number[]): Promise<void> {
343  const files = await pastedFiles($)
344  for (const [index, image] of images.entries()) {
345    const path = files.get(ids[index] ?? -1)
346    if (path !== undefined) await $.state.set({ ...IMAGE_SOURCE, id: image.id }, { path })
347  }
348}
349
350// ── The draft ───────────────────────────────────────────────────────────────
351
352async function syncDraft($: EngineInterface, text: string): Promise<void> {
353  const numbers = imageNumbers(text)
354  const key = numbers.join(',')
355  if (key === draftNumbers) return
356
357  await update($, draftImages, () => numbers.map((number) => ({ id: draftId(number), label: `Image #${number}` })))
358  // A card left open for a tag the draft no longer holds would reopen with it.
359  await update($, hint, (open) => {
360    const number = open === null ? undefined : draftNumber(open.imageId)
361    return number !== undefined && !numbers.includes(number) ? null : open
362  })
363  // Kept once both writes landed: after a failed one the next tick writes again.
364  draftNumbers = key
365}
366
367async function readDraft($: EngineInterface): Promise<void> {
368  if (isReadingDraft) return
369  isReadingDraft = true
370  try {
371    await syncDraft($, (await $.prompt.read()).text)
372  } catch {
373    // The next tick reads it again.
374  } finally {
375    isReadingDraft = false
376  }
377}
378
379// ── Presses ─────────────────────────────────────────────────────────────────
380
381async function toggleHint($: EngineInterface, site: string, imageId: string): Promise<void> {
382  if (failed.delete(imageId)) decoded.delete(imageId)
383  await update($, hint, (open) => (open?.site === site && open.imageId === imageId ? null : { site, imageId }))
384}
385
386async function closeHint($: EngineInterface): Promise<void> {
387  await update($, hint, () => null)
388}
389
390/** Opens the whole picture in the system's own viewer, which shows what cells cannot. */
391async function openOutside($: EngineInterface, image: ImageRef): Promise<void> {
392  try {
393    const source = await sourceOf($, image.id)
394    const argv = [...UV, helperPath($, 'open_image.py')]
395    const result =
396      'path' in source
397        ? await $.process.run([...argv, '--path', source.path], { timeoutMs: 30_000 })
398        : await $.process.run([...argv, '--media-type', source.mediaType], { stdin: source.base64, timeoutMs: 30_000 })
399    if (result.exitCode !== 0) $.ui.toast(`Cannot open ${image.label}: ${firstLine(result.stderr)}`)
400  } catch (error) {
401    $.ui.toast(`Cannot open ${image.label}: ${error instanceof Error ? error.message : String(error)}`)
402  }
403}
404
405// ── Drawing ─────────────────────────────────────────────────────────────────
406
407async function hintCard($: EngineInterface, site: Site, image: ImageRef, room: CellSize): Promise<RenderElement> {
408  const { Box, Text, Button, Raster } = $.ui.resolve(site)
409  const preview = await loadPreview($, image.id)
410  const size = 'decoded' in preview ? fitCells(preview.decoded.width, preview.decoded.height, room) : room
411  const inner = Math.max(size.columns, CARD_MIN_COLUMNS)
412
413  return (
414    <Box flexDirection="column" borderStyle="round" borderColor={ACCENT} paddingX={1} width={inner + 4} marginLeft={2}>
415      <Box flexDirection="row" justifyContent="space-between">
416        <Text bold wrap="truncate-end">{caption(image, preview)}</Text>
417        <Box flexDirection="row" gap={1}>
418          <Button key={`outside:${image.id}`} label="↗" plain onPress={() => openOutside($, image)} />
419          <Button key={`close:${image.id}`} label="✕" plain onPress={() => closeHint($)} />
420        </Box>
421      </Box>
422      {'decoded' in preview ? (
423        <Box justifyContent="center">
424          <Raster key={`hint:${image.id}`} columns={size.columns} rows={size.rows} cells={cellsFor(preview.decoded, size)} />
425        </Box>
426      ) : (
427        <Text dimColor wrap="wrap">{preview.problem}</Text>
428      )}
429    </Box>
430  )
431}
432
433async function chipStrip($: EngineInterface, site: Site, images: ImageRef[], lead: string, room: CellSize): Promise<RenderElement> {
434  const { Box, Text, Button } = $.ui.resolve(site)
435  const open = await read($, hint)
436  const shown = open?.site === site.requestId ? images.find((image) => image.id === open.imageId) : undefined
437  const card = shown ? await hintCard($, site, shown, room) : null
438
439  return (
440    <Box flexDirection="column">
441      <Box flexDirection="row" gap={1}>
442        <Text dimColor>{lead}</Text>
443        {images.map((image) => (
444          <Button
445            key={`chip:${image.id}`}
446            label={image.label}
447            variant={image === shown ? 'primary' : undefined}
448            onPress={() => toggleHint($, site.requestId, image.id)}
449          />
450        ))}
451      </Box>
452      {card}
453    </Box>
454  )
455}
456
457export const register: Register = (on) => {
458  on('session.start', async ($, e, next) => {
459    // A draft is a person's at the terminal prompt, and the band is drawn there alone.
460    if (e.isInteractive && e.surface === 'terminal') draftPoll ??= $.clock.every(POLL_MS, () => void readDraft($))
461    return next(e)
462  })
463
464  // `/clear`, `/resume` and a fork switch sessions without another `session.start`.
465  on('classic.SessionStart', async ($, e, next) => {
466    forgetDrafts()
467    noteTranscript(e)
468    return next(e)
469  }).catch(($, e, next) => next(e))
470
471  // Raised before the prompt's row is appended, so its relabel knows the transcript.
472  on('classic.UserPromptSubmit', async ($, e, next) => {
473    noteTranscript(e)
474    return next(e)
475  }).catch(($, e, next) => next(e))
476
477  // Kept by the row's uuid before the row is stored, as the event allows; a
478  // failure here never keeps the row from being stored.
479  on('session.append', { door: 'prompt' }, async ($, e, next) => {
480    if (e.agentId === undefined) await rememberImages($, e.uuid, e.message.content)
481    return next(e)
482  }).catch(($, e, next) => next(e))
483
484  // Paint only: the draft's chips come from the timer, since a paste raises no edit.
485  on('prompt.edit', async ($, e, next) => {
486    const box = await next(e)
487    const marks = placeholders(box.text).map(({ start, end }) => ({ start, end, color: ACCENT, underline: true }))
488    if (marks.length === 0) return box
489    return { ...box, decorations: [...(box.decorations ?? []), ...marks] }
490  }).catch(($, e, next) => next(e))
491
492  on('ui.render', { component: 'UserMessage', surface: 'terminal' }, async ($, e, next) => {
493    const images = await read($, memberOf(MESSAGE_IMAGES, e))
494    if (!images?.length) return next(e)
495
496    const { Box } = $.ui.resolve(e)
497    const drawn = await next(e)
498    const strip = await chipStrip($, e, images, '⎿', cardRoom((e.viewport?.columns ?? 80) - 2, HINT_ROOM.rows))
499    return (
500      <Box flexDirection="column">
501        {drawn}
502        <Box marginLeft={2}>{strip}</Box>
503      </Box>
504    )
505  })
506
507  on('ui.render', { component: 'AbovePrompt', surface: 'terminal' }, async ($, e, next) => {
508    const images = await read($, draftImages)
509    if (e.props.hasSurvey || images.length === 0) return next(e)
510
511    const { Box } = $.ui.resolve(e)
512    const below = await next(e)
513    // The chip row, the card's border and its header take four of the rows.
514    const strip = await chipStrip($, e, images, 'Pasted', cardRoom(e.props.bodyColumns, e.props.maxRows - 4))
515    return (
516      <Box flexDirection="column">
517        {below}
518        {strip}
519      </Box>
520    )
521  })
522}
523

1/**
2 * Pixels to terminal cells. Pure: no `$`, so the tests call it directly.
3 *
4 * A Raster cell is one `▀`: its foreground paints the upper pixel, its
5 * background the lower one. A cell is about twice as tall as it is wide, so a
6 * cell holds one square pixel above another and the picture keeps its shape.
7 */
8
9/** What scripts/image.py decoded: RGBA pixels, 4 bytes each, row-major. */
10export type Decoded = {
11  width: number
12  height: number
13  originalWidth: number
14  originalHeight: number
15  rgba: Uint8Array
16}
17
18export type CellSize = { columns: number; rows: number }
19
20const UPPER_HALF = 0x2580
21/** Raster's "terminal default" color: what a transparent pixel shows. */
22const DEFAULT_COLOR = 0x01000000
23/** How far sharpening moves a color away from its neighbours' mean, as a share of the difference. */
24const SHARPEN = 0.5
25
26/** Reads the decode helper's output: a size line, then base64 pixels. */
27export function parseDecoded(stdout: string): Decoded {
28  const newline = stdout.indexOf('\n')
29  const sizes = stdout.slice(0, newline).trim().split(' ').map(Number)
30  const [originalWidth = 0, originalHeight = 0, width = 0, height = 0] = sizes
31  const rgba = Uint8Array.fromBase64(stdout.slice(newline + 1).trim())
32
33  if (newline < 0 || width < 1 || height < 1 || rgba.length !== width * height * 4) {
34    throw new Error('the decoder answered something that is not an image')
35  }
36  return { width, height, originalWidth, originalHeight, rgba }
37}
38
39/** The largest box of cells inside the room that keeps the picture's shape. */
40export function fitCells(width: number, height: number, room: CellSize): CellSize {
41  const scale = Math.min(room.columns / width, (room.rows * 2) / height)
42  return {
43    columns: clamp(Math.round(width * scale), 1, room.columns),
44    rows: clamp(Math.round((height * scale) / 2), 1, room.rows),
45  }
46}
47
48/**
49 * Raster's `cells` for the picture scaled into `size`. A picture scaled down
50 * has its edges sharpened again.
51 */
52export function rasterCells(image: Decoded, size: CellSize): string {
53  const { columns, rows } = size
54  const scaled = scalePixels(image, columns, rows * 2)
55  const pixels = image.width > columns ? sharpen(scaled, columns, rows * 2) : scaled
56
57  const words = new Uint32Array(columns * rows * 3)
58  for (let row = 0; row < rows; row++) {
59    for (let column = 0; column < columns; column++) {
60      const at = (row * columns + column) * 3
61      words[at] = UPPER_HALF
62      words[at + 1] = pixels[row * 2 * columns + column] ?? DEFAULT_COLOR
63      words[at + 2] = pixels[(row * 2 + 1) * columns + column] ?? DEFAULT_COLOR
64    }
65  }
66  return new Uint8Array(words.buffer).toBase64()
67}
68
69/** The picture as `width` by `height` colors, row-major. */
70function scalePixels(image: Decoded, width: number, height: number): Uint32Array {
71  const pixels = new Uint32Array(width * height)
72  for (let y = 0; y < height; y++) {
73    for (let x = 0; x < width; x++) pixels[y * width + x] = averageColor(image, x, y, width, height)
74  }
75  return pixels
76}
77
78/**
79 * An unsharp mask: each color moves away from the mean of its 3 x 3
80 * neighbours by SHARPEN of the difference. Transparent pixels stay, and take
81 * no part in the mean.
82 */
83function sharpen(pixels: Uint32Array, width: number, height: number): Uint32Array {
84  const sharpened = new Uint32Array(pixels.length)
85  for (let y = 0; y < height; y++) {
86    for (let x = 0; x < width; x++) {
87      const color = pixels[y * width + x] ?? DEFAULT_COLOR
88      if (color === DEFAULT_COLOR) {
89        sharpened[y * width + x] = color
90        continue
91      }
92      const mean = [0, 0, 0]
93      let count = 0
94      for (let ny = Math.max(0, y - 1); ny <= Math.min(height - 1, y + 1); ny++) {
95        for (let nx = Math.max(0, x - 1); nx <= Math.min(width - 1, x + 1); nx++) {
96          const near = pixels[ny * width + nx] ?? DEFAULT_COLOR
97          if (near === DEFAULT_COLOR) continue
98          for (const [channel, value] of rgbOf(near).entries()) mean[channel] = (mean[channel] ?? 0) + value
99          count += 1
100        }
101      }
102      const moved = rgbOf(color).map((value, channel) => value + SHARPEN * (value - (mean[channel] ?? 0) / count))
103      sharpened[y * width + x] = colorOf(moved)
104    }
105  }
106  return sharpened
107}
108
109function rgbOf(color: number): number[] {
110  return [(color >> 16) & 0xff, (color >> 8) & 0xff, color & 0xff]
111}
112
113function colorOf([red = 0, green = 0, blue = 0]: number[]): number {
114  const channel = (value: number) => clamp(Math.round(value), 0, 255)
115  return (channel(red) << 16) | (channel(green) << 8) | channel(blue)
116}
117
118/**
119 * The mean color of the source pixels under target pixel (x, y) of a
120 * `targetWidth` by `targetHeight` grid, weighted by alpha.
121 */
122function averageColor(
123  image: Decoded,
124  x: number,
125  y: number,
126  targetWidth: number,
127  targetHeight: number,
128): number {
129  const [left, right] = span(x, targetWidth, image.width)
130  const [top, bottom] = span(y, targetHeight, image.height)
131  let red = 0
132  let green = 0
133  let blue = 0
134  let alpha = 0
135
136  for (let sy = top; sy < bottom; sy++) {
137    for (let sx = left; sx < right; sx++) {
138      const at = (sy * image.width + sx) * 4
139      const a = image.rgba[at + 3] ?? 0
140      red += (image.rgba[at] ?? 0) * a
141      green += (image.rgba[at + 1] ?? 0) * a
142      blue += (image.rgba[at + 2] ?? 0) * a
143      alpha += a
144    }
145  }
146
147  const count = (right - left) * (bottom - top)
148  if (alpha < count * 128) return DEFAULT_COLOR
149  return (Math.round(red / alpha) << 16) | (Math.round(green / alpha) << 8) | Math.round(blue / alpha)
150}
151
152/** The source range [start, end) that target index `i` of `target` covers. */
153function span(i: number, target: number, source: number): [number, number] {
154  const start = Math.min(source - 1, Math.floor((i * source) / target))
155  const end = Math.max(start + 1, Math.floor(((i + 1) * source) / target))
156  return [start, Math.min(end, source)]
157}
158
159function clamp(value: number, low: number, high: number): number {
160  return Math.max(low, Math.min(high, value))
161}
162

1/** Where Claude Code wrote `[Image #N]` for a pasted image. */
2export type Placeholder = { number: number; start: number; end: number }
3
4const PLACEHOLDER = /\[Image #(\d+)\]/g
5
6/** Every `[Image #N]` in the text, in order, repeats included. */
7export function placeholders(text: string): Placeholder[] {
8  return [...text.matchAll(PLACEHOLDER)].map((match) => ({
9    number: Number(match[1]),
10    start: match.index,
11    end: match.index + match[0].length,
12  }))
13}
14
15/** The image numbers the text names, each once, in order. */
16export function imageNumbers(text: string): number[] {
17  return [...new Set(placeholders(text).map((found) => found.number))]
18}
19

1/** One image a chip stands for. */
2export type ImageRef = {
3  /** `<message uuid>:<index>` for a sent image, `draft:<n>` for paste `n` in the draft. */
4  id: string
5  /** What the chip reads: `Image #1`. */
6  label: string
7}
8
9/** A sent image's bytes, as the transcript row carried them. */
10export type ImageData = {
11  mediaType: string
12  base64: string
13}
14
15/**
16 * Where the decoder reads an image: a file Claude Code cached for a paste,
17 * or bytes the transcript row carried, kept only while no file is known.
18 */
19export type DecodeSource = { path: string } | ImageData
20
21/** The chip whose card is open: the drawing that holds it and its image. */
22export type OpenHint = {
23  site: string
24  imageId: string
25}
26
27declare module 'claude-code' {
28  interface PluginState {
29    'image-preview': {
30      /** The images a transcript message carried, by the message's uuid. */
31      messageImages: StateFamily<ImageRef[]>
32      /** The `[Image #N]` placeholders in the prompt draft, in order. */
33      draftImages: ImageRef[]
34      /** Where each sent image is read from, by its id. */
35      imageSource: StateFamily<DecodeSource>
36      /** One card at a time, under the chip that opened it. */
37      hint: OpenHint | null
38    }
39  }
40}
41