paste-preview

1import { atom, memberOf, read, update } from 'claude-code'
2import type { Elements, EngineInterface, PromptEditResult, Register } from 'claude-code'
3
4import type { Picture } from '../types'
5
6type Block = { type: string; [field: string]: unknown }
7type ImageSource = { type?: string; media_type?: string; data?: string }
8type Size = { width: number; height: number }
9type Ran = { exitCode: number | null; stdout: string }
10
11// The hooks environment has the TC39 base64 helpers; TypeScript's es2023 lib does not declare them yet.
12const fromBase64 = (base64: string) =>
13  (Uint8Array as unknown as { fromBase64: (text: string) => Uint8Array }).fromBase64(base64)
14const toBase64 = (bytes: Uint8Array) => (bytes as unknown as { toBase64: () => string }).toBase64()
15
16const MAX_COLUMNS = 40
17const MAX_ROWS = 12
18const UPPER_HALF = 0x2580
19const LOWER_HALF = 0x2584
20const SPACE = 0x20
21const DEFAULT_COLOR = 0x01000000
22
23const PASTE_ONLY = /^\[Pasted text #\d+[^\]]*\]$/
24const HAS_PASTE = /\[Pasted text #\d+[^\]]*\]/
25const PASTES = /\[Pasted text #\d+[^\]]*\]/g
26const IMAGES = /\[Image #(\d+)\]/g
27
28const CLIPBOARD_TO_PNG = [
29  'on run argv',
30  'set outFile to open for access (POSIX file (item 1 of argv)) with write permission',
31  'try',
32  'set eof outFile to 0',
33  'write (the clipboard as «class PNGf») to outFile',
34  'close access outFile',
35  'on error errorText',
36  'close access outFile',
37  'error errorText',
38  'end try',
39  'end run',
40].flatMap(line => ['-e', line])
41
42const SAVE_AS_PNG = '[Drawing.Imaging.ImageFormat]::Png'
43
44const draft = atom({ plugin: 'paste-preview', key: 'draft' } as const, [])
45const sent = atom({ plugin: 'paste-preview', key: 'sent' } as const, [])
46const fullText = atom({ plugin: 'paste-preview', key: 'fullText' } as const, '')
47
48const triedLabels = new Set<string>()
49let isWindowsHost: Promise<boolean> | undefined
50let workDirectory: Promise<string> | undefined
51let isSyncing = false
52
53const succeeded = async (run: Promise<Ran>) => (await run).exitCode === 0
54
55const sizeOf = (width?: string, height?: string): Size | null => {
56  const size = { width: Number(width ?? 0), height: Number(height ?? 0) }
57
58  return size.width > 0 && size.height > 0 ? size : null
59}
60
61const quoted = (text: string) => `'${text.replace(/'/g, "''")}'`
62
63const powershell = ($: EngineInterface, lines: string[], stdin?: string) =>
64  $.process.run(
65    [
66      'powershell.exe',
67      '-NoProfile',
68      '-NonInteractive',
69      '-STA',
70      '-Command',
71      ["$ErrorActionPreference = 'Stop'", 'Add-Type -AssemblyName System.Drawing, System.Windows.Forms', ...lines].join('; '),
72    ],
73    stdin === undefined ? {} : { stdin },
74  )
75
76const isWindows = ($: EngineInterface) => (isWindowsHost ??= $.env.get('OS').then(os => os === 'Windows_NT'))
77
78const makeDirectory = async ($: EngineInterface, path: string) =>
79  (await isWindows($))
80    ? succeeded(powershell($, [`New-Item -ItemType Directory -Force -Path ${quoted(path)} | Out-Null`]))
81    : succeeded($.process.run(['/bin/mkdir', '-p', path]))
82
83// On Windows the text crosses PowerShell's stdout as base64 of its UTF-8 bytes: the console code page would mangle accents.
84const clipboardText = async ($: EngineInterface) => {
85  if (await isWindows($)) {
86    const { exitCode, stdout } = await powershell($, [
87      '$text = [Windows.Forms.Clipboard]::GetText()',
88      '[Console]::Out.Write([Convert]::ToBase64String([Text.Encoding]::UTF8.GetBytes($text)))',
89    ])
90
91    return exitCode === 0 ? new TextDecoder().decode(fromBase64(stdout.trim())) : ''
92  }
93
94  const { exitCode, stdout } = await $.process.run(['/usr/bin/pbpaste'])
95
96  return exitCode === 0 ? stdout : ''
97}
98
99const clipboardPng = async ($: EngineInterface, png: string) =>
100  (await isWindows($))
101    ? succeeded(
102        powershell($, [
103          '$image = [Windows.Forms.Clipboard]::GetImage()',
104          'if ($null -eq $image) { exit 1 }',
105          `$image.Save(${quoted(png)}, ${SAVE_AS_PNG})`,
106        ]),
107      )
108    : succeeded($.process.run(['/usr/bin/osascript', ...CLIPBOARD_TO_PNG, png]))
109
110const savePng = async ($: EngineInterface, png: string, base64: string, mediaType: string) => {
111  if (await isWindows($)) {
112    return succeeded(
113      powershell(
114        $,
115        [
116          '$bytes = [Convert]::FromBase64String([Console]::In.ReadToEnd())',
117          `[Drawing.Image]::FromStream([IO.MemoryStream]::new($bytes)).Save(${quoted(png)}, ${SAVE_AS_PNG})`,
118        ],
119        base64,
120      ),
121    )
122  }
123
124  const raw = png.replace(/\.png$/, `.${mediaType.split('/')[1] ?? 'png'}`)
125  const isDecoded = await succeeded($.process.run(['/usr/bin/base64', '-d', '-o', raw], { stdin: base64 }))
126
127  return isDecoded && (raw === png || (await succeeded($.process.run(['/usr/bin/sips', '-s', 'format', 'png', raw, '--out', png]))))
128}
129
130const imageSize = async ($: EngineInterface, image: string) => {
131  if (await isWindows($)) {
132    const { exitCode, stdout } = await powershell($, [
133      `$image = [Drawing.Image]::FromFile(${quoted(image)})`,
134      '[Console]::Out.Write("$($image.Width) $($image.Height)")',
135    ])
136    const match = /^(\d+) (\d+)/.exec(stdout.trim())
137
138    return exitCode === 0 ? sizeOf(match?.[1], match?.[2]) : null
139  }
140
141  const { exitCode, stdout } = await $.process.run(['/usr/bin/sips', '-g', 'pixelWidth', '-g', 'pixelHeight', image])
142
143  return exitCode === 0 ? sizeOf(/pixelWidth: (\d+)/.exec(stdout)?.[1], /pixelHeight: (\d+)/.exec(stdout)?.[1]) : null
144}
145
146const toBmp = async ($: EngineInterface, png: string, bmp: string, width: number, height: number) =>
147  (await isWindows($))
148    ? succeeded(
149        powershell($, [
150          `$image = [Drawing.Image]::FromFile(${quoted(png)})`,
151          `$bitmap = [Drawing.Bitmap]::new(${width}, ${height}, [Drawing.Imaging.PixelFormat]::Format24bppRgb)`,
152          '$graphics = [Drawing.Graphics]::FromImage($bitmap)',
153          '$graphics.InterpolationMode = [Drawing.Drawing2D.InterpolationMode]::HighQualityBicubic',
154          `$graphics.DrawImage($image, 0, 0, ${width}, ${height})`,
155          `$bitmap.Save(${quoted(bmp)}, [Drawing.Imaging.ImageFormat]::Bmp)`,
156        ]),
157      )
158    : succeeded(
159        $.process.run(['/usr/bin/sips', '-s', 'format', 'bmp', '-z', String(height), String(width), png, '--out', bmp]),
160      )
161
162const directory = ($: EngineInterface) =>
163  (workDirectory ??= (async () => {
164    const tmp = ((await isWindows($)) ? await $.env.get('TEMP') : await $.env.get('TMPDIR')) ?? '/tmp'
165    const path = `${tmp.replace(/[\\/]+$/, '')}/claude-paste-preview`
166    await makeDirectory($, path)
167
168    return path
169  })())
170
171const hasKittyGraphics = async ($: EngineInterface) => {
172  const program = (await $.env.get('TERM_PROGRAM')) ?? ''
173
174  return ['ghostty', 'kitty', 'WezTerm'].includes(program) || (await $.env.get('KITTY_WINDOW_ID')) !== undefined
175}
176
177const isPerson = (kind: string) => kind === 'composer' || kind === 'bridge'
178
179const lowestBit = (mask: number) => (mask === 0 ? 0 : 31 - Math.clz32(mask & -mask))
180
181// sips writes 32-bit BMPs with channel masks (BGRA, top-down); System.Drawing writes 24-bit BGR, bottom-up.
182const decodeBmp = (bytes: Uint8Array) => {
183  const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength)
184
185  if (bytes.byteLength < 54 || view.getUint16(0, true) !== 0x4d42) {
186    return null
187  }
188
189  const offset = view.getUint32(10, true)
190  const width = view.getInt32(18, true)
191  const rawHeight = view.getInt32(22, true)
192  const bitsPerPixel = view.getUint16(28, true)
193  const hasMasks = view.getUint32(30, true) === 3 && bytes.byteLength >= 70
194  const height = Math.abs(rawHeight)
195  const stride = Math.ceil((width * bitsPerPixel) / 32) * 4
196  const masks = hasMasks
197    ? [view.getUint32(54, true), view.getUint32(58, true), view.getUint32(62, true), view.getUint32(66, true)]
198    : [0xff0000, 0xff00, 0xff, bitsPerPixel === 32 ? 0xff000000 : 0]
199
200  if (bitsPerPixel !== 24 && bitsPerPixel !== 32) {
201    return null
202  }
203
204  return (x: number, y: number) => {
205    const line = rawHeight > 0 ? height - 1 - y : y
206    const at = offset + line * stride + (x * bitsPerPixel) / 8
207
208    if (at + bitsPerPixel / 8 > bytes.byteLength) {
209      return DEFAULT_COLOR
210    }
211
212    const pixel =
213      bitsPerPixel === 32
214        ? view.getUint32(at, true)
215        : ((bytes[at + 2] ?? 0) << 16) | ((bytes[at + 1] ?? 0) << 8) | (bytes[at] ?? 0)
216    const [red, green, blue, alpha] = masks.map(mask => ((pixel & mask) >>> lowestBit(mask)) & 0xff) as [
217      number,
218      number,
219      number,
220      number,
221    ]
222    const isTransparent = masks[3] !== 0 && alpha < 128
223
224    return isTransparent ? DEFAULT_COLOR : (red << 16) | (green << 8) | blue
225  }
226}
227
228const rasterCells = async ($: EngineInterface, png: string, columns: number, rows: number) => {
229  const bmp = png.replace(/\.png$/, `-${columns}x${rows}.bmp`)
230
231  if (!(await toBmp($, png, bmp, columns, rows * 2))) {
232    return null
233  }
234
235  const { base64 } = await $.fs.read(bmp, { as: 'bytes' })
236  const pixelAt = decodeBmp(fromBase64(base64))
237
238  if (pixelAt === null) {
239    return null
240  }
241
242  const words = new Uint32Array(columns * rows * 3)
243
244  for (let row = 0; row < rows; row++) {
245    for (let column = 0; column < columns; column++) {
246      const top = pixelAt(column, row * 2)
247      const bottom = pixelAt(column, row * 2 + 1)
248      const at = (row * columns + column) * 3
249
250      if (top === DEFAULT_COLOR && bottom === DEFAULT_COLOR) {
251        words.set([SPACE, DEFAULT_COLOR, DEFAULT_COLOR], at)
252      } else if (top === DEFAULT_COLOR) {
253        words.set([LOWER_HALF, bottom, DEFAULT_COLOR], at)
254      } else {
255        words.set([UPPER_HALF, top, bottom], at)
256      }
257    }
258  }
259
260  return toBase64(new Uint8Array(words.buffer))
261}
262
263const picture = async ($: EngineInterface, png: string, label: string): Promise<Picture | null> => {
264  const size = await imageSize($, png)
265
266  if (size === null) {
267    return null
268  }
269
270  let columns = Math.min(MAX_COLUMNS, size.width)
271  let rows = Math.max(1, Math.round((columns * size.height) / size.width / 2))
272
273  if (rows > MAX_ROWS) {
274    rows = MAX_ROWS
275    columns = Math.max(1, Math.min(MAX_COLUMNS, Math.round((rows * 2 * size.width) / size.height)))
276  }
277
278  return { label, path: png, columns, rows, cells: await rasterCells($, png, columns, rows) }
279}
280
281const syncDraft = async ($: EngineInterface, text: string) => {
282  if (isSyncing) {
283    return
284  }
285
286  isSyncing = true
287
288  try {
289    const wanted = [...text.matchAll(IMAGES)].map(match => `Image #${match[1]}`)
290    const current = await read($, draft)
291    const missing = wanted.filter(label => !triedLabels.has(label) && !current.some(one => one.label === label))
292    const isStale = current.some(one => !wanted.includes(one.label))
293
294    if (missing.length === 0 && !isStale) {
295      return
296    }
297
298    missing.forEach(label => triedLabels.add(label))
299    const newest = missing.at(-1)
300    let added: Picture | null = null
301
302    if (newest !== undefined) {
303      const png = `${await directory($)}/draft-${crypto.randomUUID()}.png`
304      added = (await clipboardPng($, png)) ? await picture($, png, newest) : null
305    }
306
307    await update($, draft, list => [
308      ...list.filter(one => wanted.includes(one.label)),
309      ...(added === null ? [] : [added]),
310    ])
311  } finally {
312    isSyncing = false
313  }
314}
315
316// The preview made while typing is reused, so the message row has its pictures from its first draw.
317const sentPictures = async ($: EngineInterface, uuid: string, text: string, images: Block[], drafts: Picture[]) => {
318  const numbers = [...text.matchAll(IMAGES)].map(match => match[1])
319  const pictures: Picture[] = []
320
321  for (const [index, block] of images.entries()) {
322    const label = `Image #${numbers[index] ?? index + 1}`
323    const drafted = drafts.find(one => one.label === label)
324    const source = block.source as ImageSource | undefined
325
326    if (drafted !== undefined) {
327      pictures.push(drafted)
328    } else if (source?.type === 'base64' && typeof source.data === 'string') {
329      const png = `${await directory($)}/${uuid}-${index}.png`
330      const isSaved = await savePng($, png, source.data, source.media_type ?? 'image/png')
331      const one = isSaved ? await picture($, png, label) : null
332
333      if (one !== null) {
334        pictures.push(one)
335      }
336    }
337  }
338
339  return pictures
340}
341
342const expandCollapsed = (before: string, box: PromptEditResult, pasted: string): PromptEditResult => {
343  if (HAS_PASTE.test(pasted)) {
344    return box
345  }
346
347  const previous = new Set(before.match(PASTES) ?? [])
348  const fresh = (box.text.match(PASTES) ?? []).find(one => !previous.has(one))
349
350  if (fresh === undefined) {
351    return box
352  }
353
354  const at = box.text.indexOf(fresh)
355
356  return {
357    text: box.text.slice(0, at) + pasted + box.text.slice(at + fresh.length),
358    cursor: at + pasted.length,
359  }
360}
361
362const pictureRow = (ui: Elements['terminal'], pictures: Picture[], isKitty: boolean) => {
363  const { Box, Text, Image, Raster } = ui
364
365  return (
366    <Box flexDirection="row" gap={2}>
367      {pictures.map(one => (
368        <Box key={one.path} flexDirection="column">
369          <Text dimColor>{one.label}</Text>
370          {isKitty || one.cells === null ? (
371            <Image source={{ file: one.path, format: 'png' }} columns={one.columns} rows={one.rows} alt={one.label} />
372          ) : (
373            <Raster key={one.path} columns={one.columns} rows={one.rows} cells={one.cells} />
374          )}
375        </Box>
376      ))}
377    </Box>
378  )
379}
380
381export const register: Register = on => {
382  on('session.start', async ($, e, next) => {
383    $.clock.every(1000, () => {
384      void $.prompt.read().then(box => syncDraft($, box.text))
385    })
386
387    return next(e)
388  })
389
390  on('prompt.edit', async ($, e, next) => {
391    const isPaste = e.key === undefined && e.inputText !== ''
392    const clip = isPaste && PASTE_ONLY.test(e.inputText.trim()) ? await clipboardText($) : ''
393    const pasted = clip.trim() === '' ? e.inputText : clip.replace(/\r\n?/g, '\n')
394    const box = await next(pasted === e.inputText ? e : { ...e, inputText: pasted })
395    const result = isPaste ? expandCollapsed(e.text, box, pasted) : box
396    await syncDraft($, result.text)
397
398    return result
399  }).catch(($, e, next) => next(e))
400
401  on('session.append', { door: 'prompt' }, async ($, e, next) => {
402    const stored = await next(e)
403
404    if (!isPerson(e.origin.kind) || e.agentId !== undefined || stored.uuid === undefined) {
405      return stored
406    }
407
408    const uuid = stored.uuid
409    const blocks: Block[] = stored.message.content
410    const first = blocks.find(block => block.type === 'text')
411    const text = typeof first?.text === 'string' ? first.text : ''
412    const images = blocks.filter(block => block.type === 'image')
413    const drafts = await read($, draft)
414
415    await update($, memberOf(fullText, { requestId: uuid }), () => text)
416    await update($, draft, () => [])
417    triedLabels.clear()
418
419    if (images.length > 0) {
420      const pictures = await sentPictures($, uuid, text, images, drafts)
421
422      if (pictures.length > 0) {
423        await update($, memberOf(sent, { requestId: uuid }), () => pictures)
424      }
425    }
426
427    return stored
428  }).catch(($, e, next) => next(e))
429
430  on('session.append', { door: 'attachment' }, async ($, e, next) => {
431    const stored = await next(e)
432
433    if (stored.uuid === undefined || stored.message.name !== 'queued_command' || e.agentId !== undefined) {
434      return stored
435    }
436
437    const uuid = stored.uuid
438    const blocks: Block[] = e.message.content
439    const images = blocks.filter(block => block.type === 'image')
440
441    if (images.length > 0) {
442      const text = blocks.flatMap(block => (typeof block.text === 'string' ? [block.text] : [])).join('\n')
443      const pictures = await sentPictures($, uuid, text, images, await read($, draft))
444
445      if (pictures.length > 0) {
446        await update($, memberOf(sent, { requestId: uuid }), () => pictures)
447      }
448    }
449
450    return stored
451  }).catch(($, e, next) => next(e))
452
453  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
454    const beneath = await next(e)
455
456    if (e.surface !== 'terminal' || e.props.hasSurvey) {
457      return beneath
458    }
459
460    const pictures = await read($, draft)
461
462    if (pictures.length === 0) {
463      return beneath
464    }
465
466    const ui = $.ui.resolve(e)
467    const { Box } = ui
468
469    return (
470      <Box flexDirection="column">
471        {beneath}
472        {pictureRow(ui, pictures, await hasKittyGraphics($))}
473      </Box>
474    )
475  })
476
477  on('ui.render', { component: 'UserMessage' }, async ($, e, next) => {
478    if (!isPerson(e.props.origin.kind)) {
479      return next(e)
480    }
481
482    const full = HAS_PASTE.test(e.props.text) ? await read($, memberOf(fullText, e)) : ''
483    const pictures = await read($, memberOf(sent, e))
484    const row = await next(full === '' ? e : { ...e, props: { ...e.props, text: full } })
485
486    if (e.surface !== 'terminal' || pictures.length === 0) {
487      return row
488    }
489
490    const ui = $.ui.resolve(e)
491    const { Box } = ui
492
493    return (
494      <Box flexDirection="column">
495        {row}
496        {pictureRow(ui, pictures, await hasKittyGraphics($))}
497      </Box>
498    )
499  })
500}
501

1export type Picture = {
2  label: string
3  path: string
4  columns: number
5  rows: number
6  cells: string | null
7}
8
9declare module 'claude-code' {
10  interface PluginState {
11    'paste-preview': {
12      draft: Picture[]
13      sent: StateFamily<Picture[]>
14      fullText: StateFamily<string>
15    }
16  }
17}
18