image-view

1import type { EngineInterface, Register, RenderComponent, RenderInput } from 'claude-code'
2
3// 能显示的图片后缀；非 PNG 的由 ImageMagick 转成 PNG
4const IMAGE_EXT = /\.(png|jpe?g|gif|webp|bmp|tiff?|svg|avif|heic|ico)$/i
5// ![alt](path) / ![alt](<path with spaces>)，可带 "title"
6const MD_IMAGE = /!\[([^\]\n]*)\]\(\s*(?:<([^>\n]+)>|([^)\s]+))(?:\s+"[^"\n]*")?\s*\)/g
7const CACHE_DIR = '/tmp/cc-image-view'
8// 图片有两种交给终端的方式：
9//  file   只传文件路径，由终端自己读取解码：原图画质，不占引擎的字节预算（默认，本机终端）
10//  inline 把 PNG 的 base64 放进渲染树：终端读不到这台机器的文件时（ssh）用，受下面的预算限制
11// 引擎限制：一棵渲染树里所有 inline Image 的 source 加起来不超过 2 MiB，超了整棵树会被丢弃。
12// 按 base64 长度保守计，一次渲染（一条回复 / 一组 Read）共用这一份预算
13const TREE_IMAGE_BUDGET = 2 * 1024 * 1024
14// inline：单张 PNG 上限（解码后）。取小一点，一条回复里才放得下几张；超过就继续缩小
15const MAX_PNG_BYTES = 700 * 1024
16// inline：缩小的边长梯度
17const RESIZE_EDGES = [1200, 900, 700, 500, 350]
18// file：非 PNG 转换时的最长边，只为避免超大图拖慢终端解码
19const FILE_MAX_EDGE = 2400
20// file 来源的路径最长 3072 字节，超过就退回 inline
21const MAX_FILE_PATH_BYTES = 3072
22const MAX_COLUMNS = 80
23// 默认高度的绝对上限；实际上限随终端行数走：视口行数 / ROWS_DIVISOR，至少 MIN_ROWS
24const MAX_ROWS = 30
25const MIN_ROWS = 6
26const ROWS_DIVISOR = 3
27// 每按一次“放大/缩小”，占的行列乘或除以这个数
28const SCALE_STEP = 1.5
29const MIN_SCALE = 0.3
30// 一个 Image 最多 255 列 × 255 行
31const MAX_CELLS = 255
32
33type Prepared =
34  | { state: 'loading' }
35  | { state: 'ready'; source: PngSource; width: number; height: number }
36  | { state: 'error'; message: string }
37
38type Engine = EngineInterface
39type PngSource = { png: string } | { file: string; format: 'png'; generation: number }
40// 消息当前在不在视口里：null 表示在视口外（只在全屏布局下有值）
41type OnScreen = { first: number; last: number; of: number } | null | undefined
42// 一棵树里还能放多少 Image source（base64 长度）；每次渲染新建一份，画图时扣减
43type Budget = { left: number }
44const newBudget = (): Budget => ({ left: TREE_IMAGE_BUDGET })
45
46const onScreenOf = (props: object): OnScreen => (props as { onScreen?: OnScreen }).onScreen
47
48const basename = (p: string) => p.slice(p.lastIndexOf('/') + 1)
49
50async function identify($: Engine, path: string) {
51  const { exitCode, stdout, stderr } = await $.process.run([
52    'identify', '-format', '%w %h %m\n', `${path}[0]`,
53  ])
54  if (exitCode !== 0) throw new Error(stderr.trim() || `identify 失败 (${exitCode})`)
55  const [w, h, format] = (stdout.split('\n')[0] ?? '').split(' ')
56  return { width: Number(w), height: Number(h), format: format ?? '' }
57}
58
59// file / inline / auto；auto 在 ssh 会话里用 inline（终端读不到这台机器的文件）
60type SourceMode = 'auto' | 'file' | 'inline'
61const SOURCE_MODES: readonly SourceMode[] = ['auto', 'file', 'inline']
62const SOURCE_MODE_KEY = 'imageSource'
63let sourceMode: SourceMode | undefined
64
65const describeSource = (mode: SourceMode) => ({
66  auto: '图片来源：auto（本机终端传文件路径，ssh 会话里内联 base64）',
67  file: '图片来源：file（传文件路径，原图画质；终端必须能读到这台机器的文件）',
68  inline: '图片来源：inline（内联 base64，会缩小到总量预算内）',
69})[mode]
70
71async function loadSourceMode($: Engine): Promise<SourceMode> {
72  if (sourceMode) return sourceMode
73  const stored = await $.store.get(SOURCE_MODE_KEY)
74  sourceMode = SOURCE_MODES.find(m => m === stored) ?? 'auto'
75  return sourceMode
76}
77
78async function useInline($: Engine): Promise<boolean> {
79  const mode = await loadSourceMode($)
80  if (mode !== 'auto') return mode === 'inline'
81  // webterm（浏览器里的 xterm.js）只支持直接传输，读不到本机文件，和 ssh 一样走 inline
82  const r = await $.process.run(['sh', '-c', 'printf %s "$SSH_CONNECTION$SSH_TTY$WEBTERM_SESSION"'])
83  return r.exitCode === 0 && r.stdout.trim() !== ''
84}
85
86const hex = async (text: string) => {
87  const digest = await crypto.subtle.digest('SHA-1', new TextEncoder().encode(text))
88  return [...new Uint8Array(digest)].map(b => b.toString(16).padStart(2, '0')).join('')
89}
90
91/** 准备好终端能显示的 PNG：file 来源是一个 PNG 文件路径，inline 来源是缩小后的 base64 */
92async function toPng($: Engine, path: string): Promise<Prepared> {
93  const info = await identify($, path)
94  const { size } = await $.fs.stat(path)
95  const inline = (await useInline($)) || new TextEncoder().encode(path).length > MAX_FILE_PATH_BYTES
96
97  if (!inline) {
98    if (info.format === 'PNG') {
99      return {
100        state: 'ready', width: info.width, height: info.height,
101        source: { file: path, format: 'png', generation: size },
102      }
103    }
104    const out = `${CACHE_DIR}/${await hex(`${path}:${size}:file`)}.png`
105    await $.process.run(['mkdir', '-p', CACHE_DIR])
106    const r = await $.process.run([
107      'convert', `${path}[0]`, '-resize', `${FILE_MAX_EDGE}x${FILE_MAX_EDGE}>`, '-strip', `png:${out}`,
108    ], { timeoutMs: 60_000 })
109    if (r.exitCode !== 0) throw new Error(r.stderr.trim() || `convert 失败 (${r.exitCode})`)
110    const converted = await identify($, out)
111    return {
112      state: 'ready', width: converted.width, height: converted.height,
113      source: { file: out, format: 'png', generation: (await $.fs.stat(out)).size },
114    }
115  }
116
117  if (info.format === 'PNG' && size <= MAX_PNG_BYTES) {
118    const { base64 } = await $.fs.read(path, { as: 'bytes' })
119    return { state: 'ready', source: { png: base64 }, width: info.width, height: info.height }
120  }
121
122  const out = `${CACHE_DIR}/${await hex(`${path}:${size}`)}.png`
123  await $.process.run(['mkdir', '-p', CACHE_DIR])
124  // 逐级缩小，直到不超过单张上限
125  for (const edge of RESIZE_EDGES) {
126    const r = await $.process.run([
127      'convert', `${path}[0]`, '-resize', `${edge}x${edge}>`, '-strip', `png:${out}`,
128    ], { timeoutMs: 60_000 })
129    if (r.exitCode !== 0) throw new Error(r.stderr.trim() || `convert 失败 (${r.exitCode})`)
130    if ((await $.fs.stat(out)).size <= MAX_PNG_BYTES) break
131  }
132  const small = await identify($, out)
133  const { base64 } = await $.fs.read(out, { as: 'bytes' })
134  return { state: 'ready', source: { png: base64 }, width: small.width, height: small.height }
135}
136
137/**
138 * 按图片比例算出占多少行列：一个字符格大约宽:高 = 1:2。
139 * scale 是 1 时不超过 MAX_COLUMNS × maxRows（maxRows 随终端行数变化，见 MAX_ROWS）；
140 * 放大后只受终端宽度和 Image 的格数上限限制，比屏幕高的图跟着对话一起滚动。
141 */
142function cellsOf(
143  width: number, height: number,
144  viewportColumns: number | undefined, viewportRows: number | undefined, scale: number,
145) {
146  // 左边缩进 2 格，再留一点边
147  const widest = Math.min(MAX_CELLS, Math.max(8, (viewportColumns ?? 100) - 4))
148  const maxRows = viewportRows === undefined
149    ? MAX_ROWS
150    : Math.min(MAX_ROWS, Math.max(MIN_ROWS, Math.floor(viewportRows / ROWS_DIVISOR)))
151  const rowsFor = (columns: number) => Math.round((columns * height) / width / 2)
152  const columnsFor = (rows: number) => Math.round((rows * 2 * width) / height)
153
154  let columns = Math.min(MAX_COLUMNS, widest, Math.ceil(width / 8))
155  if (rowsFor(columns) > maxRows) columns = Math.max(1, columnsFor(maxRows))
156
157  const base = columns
158  columns = Math.round(base * scale)
159  let atLimit = false
160  if (columns >= widest) { columns = widest; atLimit = true }
161  if (rowsFor(columns) >= MAX_CELLS) { columns = columnsFor(MAX_CELLS); atLimit = true }
162  columns = Math.max(1, Math.min(MAX_CELLS, columns))
163  // applied：夹过之后实际生效的倍数，下一次放大缩小从它算起
164  return {
165    columns, rows: Math.max(1, Math.min(MAX_CELLS, rowsFor(columns))), atLimit,
166    applied: atLimit ? columns / base : scale,
167  }
168}
169
170// 默认展开方式（/image-view 设置，存在 $.store 里跨会话保留）：
171// auto 回复里的图展开、Read 的图折叠；on 全部展开；off 全部折叠
172type OpenMode = 'auto' | 'on' | 'off'
173const OPEN_MODES: readonly OpenMode[] = ['auto', 'on', 'off']
174const OPEN_MODE_KEY = 'defaultOpen'
175let openMode: OpenMode | undefined
176
177async function loadOpenMode($: Engine): Promise<OpenMode> {
178  if (openMode) return openMode
179  const stored = await $.store.get(OPEN_MODE_KEY)
180  openMode = OPEN_MODES.find(m => m === stored) ?? 'auto'
181  return openMode
182}
183
184async function defaultOpen($: Engine, kind: 'read' | 'reply'): Promise<boolean> {
185  const mode = await loadOpenMode($)
186  return mode === 'auto' ? kind === 'reply' : mode === 'on'
187}
188
189const describeMode = (mode: OpenMode) => ({
190  auto: '图片默认展开方式：auto（回复里的图片展开，Read 的图片折叠）',
191  on: '图片默认展开方式：on（全部默认展开）',
192  off: '图片默认展开方式：off（全部默认折叠）',
193})[mode]
194
195const prepared = new Map<string, Prepared>()
196// 每个图片块的展开状态；没有记录时用该块的默认值
197const expanded = new Map<string, boolean>()
198// 每个图片块的显示倍数：占的行列相对默认大小放大或缩小了多少
199const scales = new Map<string, number>()
200
201function ensure($: Engine, path: string) {
202  if (prepared.has(path)) return
203  prepared.set(path, { state: 'loading' })
204  toPng($, path)
205    .catch((err: unknown): Prepared => ({
206      state: 'error',
207      message: err instanceof Error ? err.message : String(err),
208    }))
209    .then(result => {
210      prepared.set(path, result)
211      $.ui.invalidate('ui.render')
212    })
213}
214
215async function resolvePath($: Engine, raw: string) {
216  let p = raw.replace(/^file:\/\//, '')
217  try { p = decodeURI(p) } catch {}
218  if (p.startsWith('/')) return p
219  return `${(await $.session.cwd()).replace(/\/$/, '')}/${p.replace(/^\.\//, '')}`
220}
221
222/** 一个图片块：一行按钮 + 文件名，展开时下面是图片 */
223function imageBlock(
224  $: Engine,
225  e: RenderInput<RenderComponent, 'terminal'>,
226  id: string,
227  path: string,
228  alt: string,
229  openByDefault: boolean,
230  budget: Budget,
231  onScreen: OnScreen,
232) {
233  const { Box, Text, Button, Image } = $.ui.resolve(e)
234  const open = expanded.get(id) ?? openByDefault
235  if (open) ensure($, path)
236  const p = prepared.get(path)
237  const scale = scales.get(id) ?? 1
238  const setScale = (s: number) => () => { scales.set(id, s); $.ui.invalidate('ui.render') }
239
240  let body = null
241  let controls = null
242  // 只有 inline 来源的 base64 会进渲染树、占预算；file 来源只是一个路径
243  const cost = p?.state === 'ready' && 'png' in p.source ? p.source.png.length : 0
244  if (open && p?.state === 'ready' && cost > 0 && onScreen === null) {
245    // 滚出视口：inline 图不进渲染树（终端会按上次的行数占位），滚回来时本消息会重绘
246    body = <Text dimColor>图片在视口外，滚动到这里时显示</Text>
247  } else if (open && p?.state === 'ready' && cost > budget.left) {
248    // 引擎会丢掉超过预算的整棵树，所以这张不画，让已画的几张照常显示
249    body = <Text dimColor>本条消息里的图片总量已达显示上限（{TREE_IMAGE_BUDGET >> 20} MiB），这张暂不显示；收起其他图片后再展开</Text>
250  } else if (open && p?.state === 'ready') {
251    budget.left -= cost
252    const { columns, rows, atLimit, applied } = cellsOf(
253      p.width, p.height, e.viewport?.columns, e.viewport?.rows, scale)
254    body = (
255      <Image key={`img:${id}`} source={p.source} columns={columns} rows={rows}
256        alt={`[图片 ${alt || basename(path)} ${p.width}×${p.height}]`} />
257    )
258    controls = (
259      <Box flexDirection="row" gap={1}>
260        {!atLimit && <Button key={`grow:${id}`} label="放大" dimColor
261          onPress={setScale(applied * SCALE_STEP)} />}
262        {applied / SCALE_STEP >= MIN_SCALE && columns > 1 && rows > 1 && <Button key={`shrink:${id}`}
263          label="缩小" dimColor onPress={setScale(applied / SCALE_STEP)} />}
264        {scale !== 1 && <Button key={`reset:${id}`} label="复位" dimColor onPress={setScale(1)} />}
265        <Text dimColor>{columns}×{rows} 格</Text>
266      </Box>
267    )
268  } else if (open && p?.state === 'error') {
269    body = <Text color="red">图片无法显示：{p.message}</Text>
270  } else if (open) {
271    body = <Text dimColor>正在加载图片…</Text>
272  }
273
274  return (
275    <Box key={`blk:${id}`} flexDirection="column" marginLeft={2}>
276      <Box flexDirection="row" gap={1}>
277        <Button key={`btn:${id}`} label={open ? '收起图片' : '展开图片'} dimColor
278          onPress={() => { expanded.set(id, !open); $.ui.invalidate('ui.render') }} />
279        <Text dimColor>{alt && alt !== basename(path) ? `${alt} · ` : ''}{path}</Text>
280      </Box>
281      {controls}
282      {body}
283    </Box>
284  )
285}
286
287const imageReadPath = (tool: string, input: unknown) => {
288    if (tool !== 'Read' || typeof input !== 'object' || input === null) return null
289    const p = (input as { file_path?: unknown }).file_path
290    return typeof p === 'string' && IMAGE_EXT.test(p) ? p : null
291  }
292
293export const register: Register = on => {
294  // 单独显示的 Read 行：结果下面加一个折叠的图片块
295  on('ui.render', { component: 'ToolUse' }, async ($, e, next) => {
296    const drawn = await next(e)
297    if (e.surface !== 'terminal') return drawn
298    const { tool, input, isRunning, isErrored, tool_use_id } = e.props
299    const path = imageReadPath(tool, input)
300    if (!path || isRunning || isErrored) return drawn
301    const { Box } = $.ui.resolve(e)
302    const open = await defaultOpen($, 'read')
303    return (
304      <Box flexDirection="column">
305        {drawn}
306        {imageBlock($, e, tool_use_id, path, '', open, newBudget(), onScreenOf(e.props))}
307      </Box>
308    )
309  })
310
311  // 折叠成一行的 "Read N files"：给其中每张图片各加一个折叠的图片块
312  on('ui.render', { component: 'ToolGroup' }, async ($, e, next) => {
313    const drawn = await next(e)
314    if (e.surface !== 'terminal' || e.props.isExpanded) return drawn
315    const budget = newBudget()
316    const open = await defaultOpen($, 'read')
317    const blocks = e.props.calls.flatMap((c, i) => {
318      const path = imageReadPath(c.tool, c.input)
319      if (!path || c.isRunning || c.isErrored) return []
320      return [imageBlock($, e, c.tool_use_id ?? `${e.requestId}:${i}`, path, '', open, budget, onScreenOf(e.props))]
321    })
322    if (blocks.length === 0) return drawn
323    const { Box } = $.ui.resolve(e)
324    return (
325      <Box flexDirection="column">
326        {drawn}
327        {blocks}
328      </Box>
329    )
330  })
331
332  // 回复里的 ![alt](path)：默认展开显示
333  on('ui.render', { component: 'AssistantMessage' }, async ($, e, next) => {
334    const drawn = await next(e)
335    if (e.surface !== 'terminal') return drawn
336    const found = [...e.props.text.matchAll(MD_IMAGE)]
337      .map(m => ({ alt: m[1] ?? '', raw: m[2] ?? m[3] ?? '' }))
338      .filter(m => IMAGE_EXT.test(m.raw) && !/^https?:/i.test(m.raw))
339    if (found.length === 0) return drawn
340    const budget = newBudget()
341    const open = await defaultOpen($, 'reply')
342    // 先解析完路径再依次画，预算按出现顺序扣
343    const paths = await Promise.all(found.map(m => resolvePath($, m.raw)))
344    const blocks = found.map((m, i) =>
345      imageBlock($, e, `${e.requestId}:${i}`, paths[i]!, m.alt, open, budget, onScreenOf(e.props)))
346    const { Box } = $.ui.resolve(e)
347    return (
348      <Box flexDirection="column">
349        {drawn}
350        {blocks}
351      </Box>
352    )
353  })
354
355  // /image-view [on|off|auto]：设置图片默认展开还是折叠；不带参数显示当前设置
356  on('session.start', async ($, e, next) => {
357    await $.command.register({
358      name: 'image-view',
359      description: '设置图片默认展开(on)、折叠(off)或自动(auto)；不带参数显示当前设置',
360      argumentHint: '[on|off|auto]',
361    })
362    return next(e)
363  })
364
365  on('command.run', { command: 'image-view' }, async ($, e) => {
366    const [first = '', second = ''] = e.args.trim().toLowerCase().split(/\s+/)
367    const usage = '用法：/image-view on|off|auto　　/image-view source auto|file|inline'
368    const current = await loadOpenMode($)
369    const currentSource = await loadSourceMode($)
370    if (first === '') return { text: `${describeMode(current)}\n图片来源：${currentSource}\n${usage}` }
371
372    if (first === 'source') {
373      const picked = SOURCE_MODES.find(m => m === second)
374      if (!picked) return { text: `未知参数 "${second}"。${usage}\n图片来源：${currentSource}` }
375      await $.store.set(SOURCE_MODE_KEY, picked)
376      sourceMode = picked
377      prepared.clear() // 重新按新来源准备
378      $.ui.invalidate('ui.render')
379      return { text: describeSource(picked) }
380    }
381
382    const mode = OPEN_MODES.find(m => m === first)
383    if (!mode) return { text: `未知参数 "${first}"。${usage}\n${describeMode(current)}` }
384    await $.store.set(OPEN_MODE_KEY, mode)
385    openMode = mode
386    // 已手动展开/收起的块也回到新的默认值
387    expanded.clear()
388    $.ui.invalidate('ui.render')
389    return { text: describeMode(mode) }
390  })
391
392  // 告诉模型：交付图片时用 Markdown 图片语法引用本地绝对路径，终端会直接显示
393  on('prompt.context', async ($, e, next) => {
394    const r = await next(e)
395    return {
396      ...r,
397      blocks: [
398        ...r.blocks,
399        {
400          name: 'imageView',
401          text: 'The terminal renders images inline. When you produce or want to show the user a local image file (a chart you plotted, a screenshot, a generated picture), reference it in your reply with Markdown image syntax and an absolute path, e.g. `![p95 latency](/abs/path/chart.png)`; it is drawn under your message. Images you Read get an expand button automatically, so do not re-embed those unless asked.',
402        },
403      ],
404    }
405  })
406}
407