img-view

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import type { Picture } from '../types'
5import { IMAGE, drawnBy, fit, parseBmp, pathsIn, quadrants } from './picture'
6
7const PANE = 'img-view'
8const pictures = atom({ plugin: 'img-view', key: 'pictures' } as const, [])
9const selected = atom({ plugin: 'img-view', key: 'selected' } as const, -1)
10
11// Terminals that draw the kitty graphics protocol show the pixels themselves; the rest get
12// half-blocks. IMG_VIEW=image or IMG_VIEW=blocks overrides the guess.
13const GRAPHICAL = /^(iTerm\.app|WezTerm|ghostty|kitty)$/i
14
15// The file's bytes and each size it was drawn at, kept for this load of the module.
16const pngs = new Map<string, string>()
17const blocks = new Map<string, { cells: string; columns: number; rows: number }>()
18const sizes = new Map<string, { width: number; height: number }>()
19
20async function hasPixels($: EngineInterface): Promise<boolean> {
21  const forced = await $.env.get('IMG_VIEW')
22  if (forced === 'image') return true
23  if (forced === 'blocks') return false
24  const program = (await $.env.get('TERM_PROGRAM')) ?? ''
25  const term = (await $.env.get('TERM')) ?? ''
26  return GRAPHICAL.test(program) || /kitty|ghostty/.test(term)
27}
28
29/** The picture's size in pixels, from the file itself (sips on macOS, ImageMagick elsewhere). */
30async function sizeOf($: EngineInterface, path: string): Promise<{ width: number; height: number } | undefined> {
31  const known = sizes.get(path)
32  if (known) return known
33  let width = 0
34  let height = 0
35  const sips = await $.process.run(['sips', '-g', 'pixelWidth', '-g', 'pixelHeight', path]).catch(() => undefined)
36  if (sips?.exitCode === 0) {
37    width = Number(/pixelWidth: (\d+)/.exec(sips.stdout)?.[1] ?? 0)
38    height = Number(/pixelHeight: (\d+)/.exec(sips.stdout)?.[1] ?? 0)
39  } else {
40    const magick = await $.process.run(['magick', 'identify', '-format', '%w %h', `${path}[0]`]).catch(() => undefined)
41    const [w, h] = (magick?.stdout ?? '').trim().split(' ').map(Number)
42    width = w ?? 0
43    height = h ?? 0
44  }
45  if (!width || !height) return undefined
46  sizes.set(path, { width, height })
47  return { width, height }
48}
49
50/** The picture scaled to exactly (columns·2) × (rows·2) pixels, as quadrant cells. */
51async function blocksOf($: EngineInterface, path: string, columns: number, rows: number) {
52  const key = `${path}|${columns}x${rows}`
53  const known = blocks.get(key)
54  if (known) return known
55  const tmp = `${(await $.env.get('TMPDIR')) ?? '/tmp/'}img-view-${columns}x${rows}-${path.replace(/[^\w.-]/g, '_').slice(-80)}.bmp`
56  const sips = await $.process.run(['sips', '-z', String(rows * 2), String(columns * 2), path, '-s', 'format', 'bmp', '--out', tmp]).catch(() => undefined)
57  if (sips?.exitCode !== 0) {
58    const magick = await $.process.run(['magick', `${path}[0]`, '-resize', `${columns * 2}x${rows * 2}!`, `bmp3:${tmp}`]).catch(() => undefined)
59    if (magick?.exitCode !== 0) throw new Error('neither sips nor ImageMagick could read it')
60  }
61  const { base64 } = await $.fs.read(tmp, { as: 'bytes' })
62  const drawn = quadrants(parseBmp(Uint8Array.fromBase64(base64)))
63  blocks.set(key, drawn)
64  return drawn
65}
66
67async function pngOf($: EngineInterface, path: string): Promise<string> {
68  const known = pngs.get(path)
69  if (known !== undefined) return known
70  const { base64 } = await $.fs.read(path, { as: 'bytes' })
71  pngs.set(path, base64)
72  return base64
73}
74
75async function show($: EngineInterface, found: Picture[]) {
76  if (found.length === 0) return
77  const list = await update($, pictures, old => [...old.filter(p => !found.some(f => f.path === p.path)), ...found].slice(-50))
78  await update($, selected, () => list.length - found.length)
79  $.ui.status(`${list.length} picture${list.length === 1 ? '' : 's'} — /img`)
80  void $.ui.open({ id: PANE, title: 'Pictures' })
81}
82
83export const register: Register = on => {
84  on('session.start', async ($, e, next) => {
85    await $.command.register({ name: 'img', description: 'Show an image in the pictures pane: /img <path>, or no path for the pane' })
86    return next(e)
87  })
88
89  on('command.run', { command: 'img' }, async ($, e) => {
90    const path = e.args.trim().replace(/^~(?=\/)/, (await $.env.get('HOME')) ?? '~')
91    if (path) {
92      if (!(await $.fs.exists(path))) return { text: `No file at ${path}.` }
93      await show($, [{ path, label: '/img', issues: 0 }])
94      return { text: `Showing ${path}.` }
95    }
96    await $.ui.open({ id: PANE, title: 'Pictures' })
97    return { text: 'Pictures pane opened.' }
98  })
99
100  // Pictures arrive from the tools that make or read them: AIGUI's drawings, an image file
101  // the model reads or writes, and any absolute image path an AIGUI tool reports.
102  on('tool.call', async ($, e, next) => {
103    const ran = await next(e)
104    if (ran.deny !== undefined || ran.isError) return ran
105    const input = e as unknown as { file_path?: unknown }
106    const filePath = typeof input.file_path === 'string' ? input.file_path : ''
107    if ((e.tool === 'Read' || e.tool === 'Write') && IMAGE.test(filePath)) {
108      await show($, [{ path: filePath, label: e.tool, issues: 0 }])
109    } else if (/aigui_/.test(e.tool) && ran.text) {
110      const drawn = drawnBy(ran.text)
111      const named = drawn.length ? drawn : pathsIn(ran.text).map(path => ({ path, label: 'aigui', issues: 0 }))
112      await show($, named)
113    }
114    return ran
115  })
116
117  on('ui.render', { component: 'Pane', requestId: PANE }, async ($, e) => {
118    const ui = $.ui.resolve(e)
119    const { Box, Text, Button } = ui
120    const list = await read($, pictures)
121    const at = Math.min(Math.max(await read($, selected), 0), list.length - 1)
122    const picture = list[at]
123    if (!picture) return <Text dimColor>No pictures yet. /img path shows one; AIGUI's drawings and image files Claude reads appear here.</Text>
124
125    const name = picture.path.split('/').pop() ?? picture.path
126    const alt = `${picture.label} — ${name}`
127    const go = (step: number) => () => update($, selected, () => Math.min(Math.max(at + step, 0), list.length - 1))
128    // The pane's own body, not the terminal: two rows go to the title line and the buttons.
129    const room = {
130      columns: Math.max(10, e.props.bodyColumns ?? (e.viewport?.columns ?? 80) - 2),
131      rows: Math.max(4, (e.props.scroll?.bodyRows ?? (e.viewport?.rows ?? 30) - 4) - 2),
132    }
133
134    let body
135    try {
136      const size = picture.width && picture.height ? { width: picture.width, height: picture.height } : await sizeOf($, picture.path)
137      if (!size) throw new Error('unknown size')
138      const box = fit(size.width, size.height, room)
139      if (e.surface === 'terminal') {
140        const { Image, Raster } = $.ui.resolve(e as typeof e & { surface: 'terminal' })
141        if ((await hasPixels($)) && /\.png$/i.test(picture.path)) {
142          body = <Image key="picture" source={{ png: await pngOf($, picture.path) }} columns={box.columns} rows={box.rows} alt={alt} />
143        } else {
144          const drawn = await blocksOf($, picture.path, box.columns, box.rows)
145          body = <Raster key="picture" cells={drawn.cells} columns={drawn.columns} rows={drawn.rows} />
146        }
147      } else if ('Svg' in ui && /\.(png|jpe?g|gif|webp)$/i.test(picture.path)) {
148        const { Svg } = ui as typeof ui & { Svg: (props: { source: string; alt: string }) => unknown }
149        const kind = /\.png$/i.test(picture.path) ? 'png' : /\.gif$/i.test(picture.path) ? 'gif' : /\.webp$/i.test(picture.path) ? 'webp' : 'jpeg'
150        const data = await pngOf($, picture.path)
151        const svg = `<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 ${size.width} ${size.height}"><image width="${size.width}" height="${size.height}" href="data:image/${kind};base64,${data}"/></svg>`
152        body = <Svg source={svg} alt={alt} />
153      } else {
154        body = <Text dimColor>{alt}</Text>
155      }
156    } catch (error) {
157      body = <Text dimColor>{name}: {(await $.fs.exists(picture.path)) ? String((error as Error).message) : 'no longer on disk'}.</Text>
158    }
159
160    return (
161      <Box flexDirection="column">
162        <Box>
163          <Text bold>{picture.label}</Text>
164          <Text dimColor> {at + 1}/{list.length}  {name}</Text>
165          {picture.issues > 0 && <Text color="yellow">  {picture.issues} problem{picture.issues === 1 ? '' : 's'} found</Text>}
166        </Box>
167        {body}
168        <Box>
169          <Button key="prev" hotkey="p" label="Previous" plain onPress={go(-1)} />
170          <Text> </Text>
171          <Button key="next" hotkey="n" label="Next" plain onPress={go(1)} />
172          <Text> </Text>
173          <Button key="open" hotkey="o" label="Open" plain onPress={() => $.process.run(['open', picture.path])} />
174        </Box>
175      </Box>
176    )
177  })
178}
179

1// What turns a picture into something a pane can hold, kept free of `$` so tests reach it.
2
3import type { Picture } from '../types'
4
5export const IMAGE = /\.(png|jpe?g|gif|webp|bmp|tiff?|heic)$/i
6
7/** Uncompressed pixels, top row first, 4 bytes each (r, g, b, a). */
8export type Pixels = { width: number; height: number; rgba: Uint8Array }
9
10/**
11 * A BMP as `sips -s format bmp` and `magick … bmp3:` write it: BITMAPINFOHEADER or later,
12 * 24 or 32 bits a pixel, uncompressed or bit-field masks, rows bottom-up or top-down.
13 */
14export function parseBmp(bytes: Uint8Array): Pixels {
15  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
16  if (bytes[0] !== 0x42 || bytes[1] !== 0x4d) throw new Error('not a BMP')
17  const offset = view.getUint32(10, true)
18  const header = view.getUint32(14, true)
19  const width = view.getInt32(18, true)
20  const signed = view.getInt32(22, true)
21  const bits = view.getUint16(28, true)
22  const compression = view.getUint32(30, true)
23  if (bits !== 24 && bits !== 32) throw new Error(`${bits}-bit BMP`)
24  if (compression !== 0 && compression !== 3) throw new Error('compressed BMP')
25  const height = Math.abs(signed)
26  const isTopDown = signed < 0
27  // Bit-field masks follow a 40-byte header (compression 3) or sit inside a longer one.
28  const masks = compression === 3 && header >= 40
29    ? [0, 1, 2, 3].map(i => (header >= 56 || i < 3 ? view.getUint32(54 + i * 4, true) : 0))
30    : bits === 32 ? [0x00ff0000, 0x0000ff00, 0x000000ff, 0xff000000] : [0, 0, 0, 0]
31  const stride = Math.ceil((width * bits) / 32) * 4
32  const rgba = new Uint8Array(width * height * 4)
33  const channel = (value: number, mask: number) => {
34    if (mask === 0) return 255
35    let shift = 0
36    while (((mask >>> shift) & 1) === 0) shift += 1
37    return Math.round((((value & mask) >>> shift) * 255) / (mask >>> shift))
38  }
39  for (let y = 0; y < height; y += 1) {
40    const row = offset + (isTopDown ? y : height - 1 - y) * stride
41    for (let x = 0; x < width; x += 1) {
42      const at = (y * width + x) * 4
43      if (bits === 24) {
44        const p = row + x * 3
45        rgba[at] = bytes[p + 2] ?? 0
46        rgba[at + 1] = bytes[p + 1] ?? 0
47        rgba[at + 2] = bytes[p] ?? 0
48        rgba[at + 3] = 255
49      } else {
50        const value = view.getUint32(row + x * 4, true)
51        rgba[at] = channel(value, masks[0] ?? 0)
52        rgba[at + 1] = channel(value, masks[1] ?? 0)
53        rgba[at + 2] = channel(value, masks[2] ?? 0)
54        rgba[at + 3] = masks[3] ? channel(value, masks[3]) : 255
55      }
56    }
57  }
58  return { width, height, rgba }
59}
60
61const DEFAULT = 0x01000000
62const FULL = 0x2588
63
64// The glyph that inks the quarters a mask names in the foreground: bit 8 top-left,
65// 4 top-right, 2 bottom-left, 1 bottom-right. Every one is a width-1 BMP character.
66const QUADRANTS = [
67  FULL, 0x2597, 0x2596, 0x2584, 0x259d, 0x2590, 0x259e, 0x259f,
68  0x2598, 0x259a, 0x258c, 0x2599, 0x2580, 0x259c, 0x259b, FULL,
69]
70
71/**
72 * Two by two pixels a cell, the way chafa draws without a graphics protocol: of the
73 * fourteen ways to split a cell's four pixels into two groups, the one whose two average
74 * colours sit closest to the pixels, drawn as the quadrant glyph of that split. A pixel
75 * under half opaque is left to the terminal's own background.
76 */
77export function quadrants(pixels: Pixels): { cells: string; columns: number; rows: number } {
78  const columns = Math.ceil(pixels.width / 2)
79  const rows = Math.ceil(pixels.height / 2)
80  const words = new Uint32Array(columns * rows * 3)
81  const quad = [0, 0, 0, 0].map(() => [0, 0, 0, 0])
82  const read = (x: number, y: number, into: number[]) => {
83    const cx = Math.min(x, pixels.width - 1)
84    const cy = Math.min(y, pixels.height - 1)
85    const at = (cy * pixels.width + cx) * 4
86    into[0] = pixels.rgba[at] ?? 0
87    into[1] = pixels.rgba[at + 1] ?? 0
88    into[2] = pixels.rgba[at + 2] ?? 0
89    into[3] = pixels.rgba[at + 3] ?? 0
90  }
91  const pack = (r: number, g: number, b: number) => (Math.round(r) << 16) | (Math.round(g) << 8) | Math.round(b)
92  for (let row = 0; row < rows; row += 1) {
93    for (let column = 0; column < columns; column += 1) {
94      // Quarters in mask order: top-left, top-right, bottom-left, bottom-right.
95      read(column * 2, row * 2, quad[0] as number[])
96      read(column * 2 + 1, row * 2, quad[1] as number[])
97      read(column * 2, row * 2 + 1, quad[2] as number[])
98      read(column * 2 + 1, row * 2 + 1, quad[3] as number[])
99      const at = (row * columns + column) * 3
100      const opaque = quad.map(q => (q[3] ?? 0) >= 128)
101      if (!opaque.every(Boolean)) {
102        // Part see-through: ink the opaque quarters in their average, the rest left bare.
103        const ink = quad.filter((_, i) => opaque[i])
104        const mask = opaque.reduce((m, o, i) => (o ? m | (8 >> i) : m), 0)
105        const avg = (c: number) => ink.reduce((sum, q) => sum + (q[c] ?? 0), 0) / Math.max(1, ink.length)
106        words[at] = mask === 0 ? FULL : (QUADRANTS[mask] ?? FULL)
107        words[at + 1] = mask === 0 ? DEFAULT : pack(avg(0), avg(1), avg(2))
108        words[at + 2] = DEFAULT
109        continue
110      }
111      let best = { error: Infinity, mask: 15, fg: 0, bg: 0 }
112      for (let mask = 1; mask < 16; mask += 1) {
113        const sums = [[0, 0, 0, 0], [0, 0, 0, 0]] as [number[], number[]]
114        quad.forEach((q, i) => {
115          const side = sums[(mask & (8 >> i)) ? 0 : 1]
116          side[0] = (side[0] ?? 0) + (q[0] ?? 0)
117          side[1] = (side[1] ?? 0) + (q[1] ?? 0)
118          side[2] = (side[2] ?? 0) + (q[2] ?? 0)
119          side[3] = (side[3] ?? 0) + 1
120        })
121        const mean = sums.map(s => [0, 1, 2].map(c => (s[c] ?? 0) / Math.max(1, s[3] ?? 0)))
122        let error = 0
123        quad.forEach((q, i) => {
124          const m = mean[(mask & (8 >> i)) ? 0 : 1] as number[]
125          for (let c = 0; c < 3; c += 1) error += ((q[c] ?? 0) - (m[c] ?? 0)) ** 2
126        })
127        if (error < best.error) {
128          const [f, b] = mean as [number[], number[]]
129          best = { error, mask, fg: pack(f[0] ?? 0, f[1] ?? 0, f[2] ?? 0), bg: pack(b[0] ?? 0, b[1] ?? 0, b[2] ?? 0) }
130        }
131      }
132      // One colour after all: a full block, which no font leaves a seam in.
133      const isFlat = best.mask === 15 || best.fg === best.bg
134      words[at] = isFlat ? FULL : (QUADRANTS[best.mask] ?? FULL)
135      words[at + 1] = best.fg
136      words[at + 2] = isFlat ? best.fg : best.bg
137    }
138  }
139  return { cells: new Uint8Array(words.buffer).toBase64(), columns, rows }
140}
141
142/**
143 * The box a w×h picture takes: as wide as the room allows, a cell being about twice as
144 * tall as it is wide, shrunk to the room's height when that runs out first.
145 */
146export function fit(width: number, height: number, room: { columns: number; rows: number }) {
147  let columns = Math.min(room.columns, 160)
148  let rows = Math.round((columns * height) / width / 2)
149  if (rows > room.rows) {
150    rows = room.rows
151    columns = Math.round((rows * 2 * width) / height)
152  }
153  return { columns: Math.max(1, Math.min(columns, 255)), rows: Math.max(1, Math.min(rows, 255)) }
154}
155
156// aigui_render's summary names each picture on its own line, its problems indented under it:
157//   - chart: /Users/me/.cache/aigui/images/chart-1a2b.png (1200×800)
158//     ! two labels overlap
159const DRAWN = /^- ([\w-]+): (\/.+\.png) \((\d+)×(\d+)\)$/
160
161export function drawnBy(text: string): Picture[] {
162  const found: Picture[] = []
163  for (const line of text.split('\n')) {
164    const hit = DRAWN.exec(line)
165    const last = found[found.length - 1]
166    if (hit) found.push({ label: hit[1] ?? 'aigui', path: hit[2] ?? '', width: Number(hit[3]), height: Number(hit[4]), issues: 0 })
167    else if (line.startsWith('  ! ') && last) last.issues += 1
168  }
169  return found
170}
171
172/** `Saved /path/x.png` and the like: every absolute image path a tool's text names. */
173export function pathsIn(text: string): string[] {
174  const found = text.match(/\/[^\s'"`()<>]+\.(?:png|jpe?g|gif|webp|bmp|tiff?|heic)\b/gi) ?? []
175  return [...new Set(found)]
176}
177

1/** One picture the pane can show: a file on this machine, with what is known about it. */
2export type Picture = {
3  path: string
4  /** What drew it or where it came from: `chart`, `Read`, `/img`. */
5  label: string
6  width?: number
7  height?: number
8  /** Problems AIGUI found in the drawing. */
9  issues: number
10}
11
12declare module 'claude-code' {
13  interface PluginState {
14    'img-view': { pictures: Picture[]; selected: number }
15  }
16}
17