media-view

1import { atom, read, update } from 'claude-code'
2import type { EngineInterface, Register } from 'claude-code'
3
4import { stopAnimations } from './adapters/animator'
5import { enginePorts } from './adapters/engine'
6import { drawAbovePrompt, drawMediaViewPane, drawPictureRows } from './adapters/views'
7import { openCommand } from './domain/platform'
8import { mediaViewClosed } from './usecases/closes/mediaView'
9import { mediaView } from './usecases/commandRuns/mediaView'
10import { rememberReadPath } from './usecases/images'
11import { abovePrompt, watchDraft } from './usecases/renders/abovePrompt'
12import { MEDIA_VIEW_PANE, mediaViewPane } from './usecases/renders/pane/mediaView'
13import { toolGroup } from './usecases/renders/toolGroup'
14import { toolResult } from './usecases/renders/toolResult'
15import { userMessage } from './usecases/renders/userMessage'
16import { sessionStarts } from './usecases/sessionStarts'
17import { frames } from './usecases/toolCalls/frames'
18import type { Ports } from './usecases/ports'
19
20const TRANSCRIPT_SITES = ['ToolGroup', 'ToolResult', 'UserMessage'] as const
21
22// State refs live where they are read and written (ADR 0014); types/index.d.ts lists them.
23const isTranscriptAnimated = atom({ plugin: 'media-view', key: 'isTranscriptAnimated' } as const, true)
24const draftImageNumbers = atom({ plugin: 'media-view', key: 'draftImageNumbers' } as const, [])
25const mediaViewScrollOffset = atom({ plugin: 'media-view', key: 'mediaViewScrollOffset' } as const, 0)
26
27// Ports over the session's $, made at session.start: work they start outlives the render that asked for it,
28// which a newer draw of the same row would otherwise cut short (ADR 0007). Without a session (tests that only
29// draw), a hook's own $ serves.
30let sessionPorts: Ports | undefined
31const portsFor = ($: EngineInterface) => sessionPorts ?? enginePorts($)
32
33// The composition root: every hook, each answered by a usecase over the engine's ports (ADRs 0014, 0015).
34export const register: Register = on => {
35  on('session.start', async ($, e, next) => {
36    const started = await next(e)
37    sessionPorts = enginePorts($)
38    await sessionStarts({
39      watchDraft: () =>
40        watchDraft({
41          every: (ms, tick) => $.clock.every(ms, () => void tick()),
42          readDraft: async () => (await $.prompt.read()).text,
43          // `change` hands back `current` itself when nothing changed: then nothing is written.
44          updateDraftNumbers: async change => {
45            const current = await read($, draftImageNumbers)
46            const changed = change(current)
47            if (changed !== current) {
48              await update($, draftImageNumbers, () => [...changed])
49            }
50          },
51        }),
52      registerTool: async tool => {
53        await $.tool.register({ ...tool, isDeferred: false })
54      },
55      registerCommand: async command => {
56        await $.command.register(command)
57      },
58    })
59    return started
60  })
61
62  on('ui.render', { component: 'AbovePrompt' }, async ($, e, next) => {
63    if (e.surface !== 'terminal') {
64      return next(e)
65    }
66    const numbers = await read($, draftImageNumbers)
67    const view = e.props.hasSurvey
68      ? null
69      : await abovePrompt(portsFor($), { numbers, bodyColumns: e.props.bodyColumns, maxRows: e.props.maxRows })
70    if (view === null) {
71      stopAnimations('AbovePrompt', e.requestId)
72      return next(e)
73    }
74    return drawAbovePrompt($, e, view)
75  })
76
77  on('ui.render', { component: 'ToolGroup' }, async ($, e, next) => {
78    if (e.surface !== 'terminal') {
79      return next(e)
80    }
81    const rows = await toolGroup(portsFor($), { requestId: e.requestId, calls: e.props.calls, viewportColumns: e.viewport?.columns })
82    const own = await next(e)
83    return rows === null ? own : drawPictureRows($, e, { site: 'ToolGroup', own, rows, isAnimated: await read($, isTranscriptAnimated) })
84  })
85
86  on('ui.render', { component: 'ToolResult' }, async ($, e, next) => {
87    if (e.surface !== 'terminal') {
88      return next(e)
89    }
90    const rows = await toolResult(portsFor($), {
91      toolUseId: e.props.tool_use_id,
92      tool: e.props.tool,
93      output: e.props.output,
94      viewportColumns: e.viewport?.columns,
95    })
96    const own = await next(e)
97    return rows === null ? own : drawPictureRows($, e, { site: 'ToolResult', own, rows, isAnimated: await read($, isTranscriptAnimated) })
98  })
99
100  on('ui.render', { component: 'UserMessage' }, async ($, e, next) => {
101    if (e.surface !== 'terminal') {
102      return next(e)
103    }
104    const rows = await userMessage(portsFor($), {
105      requestId: e.requestId,
106      text: e.props.text,
107      isFromSomeoneElse: e.props.from !== undefined,
108      viewportColumns: e.viewport?.columns,
109    })
110    const own = await next(e)
111    return rows === null ? own : drawPictureRows($, e, { site: 'UserMessage', own, rows, isAnimated: await read($, isTranscriptAnimated) })
112  })
113
114  on('ui.render', { component: 'Pane', requestId: MEDIA_VIEW_PANE }, async ($, e) => {
115    if (e.surface !== 'terminal') {
116      const { Text } = $.ui.resolve(e)
117      return <Text>media-view draws pictures in the terminal.</Text>
118    }
119    // Read so that a scroll of this pane (written below) redraws this pane alone.
120    await read($, mediaViewScrollOffset)
121    const ports = portsFor($)
122    return drawMediaViewPane($, e, await mediaViewPane(ports, { bodyColumns: e.props.bodyColumns, scroll: e.props.scroll }), {
123      copyPath: path => $.ui.copy({ text: path }),
124      open: async path => ports.run(openCommand((await ports.host()).os, path)),
125      showInChat: rowId => $.ui.scroll({ to: { requestId: rowId }, block: 'center' }),
126    })
127  })
128
129  on('ui.scroll', { requestId: MEDIA_VIEW_PANE }, async ($, e, next) => {
130    const scrolled = await next(e)
131    await update($, mediaViewScrollOffset, () => e.offset)
132    return scrolled
133  })
134
135  on('ui.close', ($, e, next) => {
136    mediaViewClosed(e.id, { stopPaneAnimations: () => stopAnimations('Pane') })
137    return next(e)
138  })
139
140  on('command.run', { command: 'media-view' }, async ($, e) => ({
141    text: await mediaView(
142      {
143        openPane: async () => {
144          await $.ui.open({ id: MEDIA_VIEW_PANE, title: 'Media' })
145        },
146        setTranscriptAnimated: async isAnimated => {
147          await update($, isTranscriptAnimated, () => isAnimated)
148          if (!isAnimated) {
149            TRANSCRIPT_SITES.forEach(site => stopAnimations(site))
150          }
151        },
152      },
153      e.args,
154    ),
155  }))
156
157  on('tool.call', { tool: 'mcp__media-view__frames' }, async ($, e) => frames(portsFor($), e))
158
159  // A ToolResult row's props carry no path; the Read call's input does.
160  on('tool.call', { tool: 'Read' }, ($, e, next) => {
161    rememberReadPath(e.tool_use_id, e.file_path)
162    return next(e)
163  })
164}
165

1import type { EngineInterface, Timer } from 'claude-code'
2
3import { isTerminalDeny, noticeFor } from '../domain/blitDeny'
4import type { Frame } from '../domain/media/frameCache'
5
6// A blit sent in the same pass as the render that mounts its Image lands before the mount.
7const RETRIES_BEFORE_MOUNT = 20
8const RETRY_MS = 100
9
10export type Site = 'AbovePrompt' | 'ToolGroup' | 'ToolResult' | 'UserMessage' | 'Pane'
11
12// One blit loop; a stopped loop's tick that was waiting on a blit sees `isStopped` and ends.
13type Loop = { timer: Timer | undefined; isStopped: boolean }
14
15const loops = new Map<string, Loop>()
16const checkedStills = new Set<string>()
17let isTerminalRefusing = false
18
19const loopId = (site: Site, requestId: string, key: string) => `${site}/${requestId}/${key}`
20
21const stop = (id: string, loop: Loop) => {
22  loop.isStopped = true
23  loop.timer?.cancel()
24  loops.delete(id)
25}
26
27export const stopAnimation = (site: Site, requestId: string, key: string) => {
28  const id = loopId(site, requestId, key)
29  const loop = loops.get(id)
30  if (loop !== undefined) {
31    stop(id, loop)
32  }
33}
34
35export const stopAnimations = (site: Site, requestId?: string) => {
36  const prefix = requestId === undefined ? `${site}/` : `${site}/${requestId}/`
37  for (const [id, loop] of loops) {
38    if (id.startsWith(prefix)) {
39      stop(id, loop)
40    }
41  }
42}
43
44// Stops the loops of a site's request whose Images are no longer drawn.
45export const stopAnimationsExcept = (site: Site, requestId: string, keys: readonly string[]) => {
46  const drawn = new Set(keys.map(key => loopId(site, requestId, key)))
47  for (const [id, loop] of loops) {
48    if (id.startsWith(`${site}/${requestId}/`) && !drawn.has(id)) {
49      stop(id, loop)
50    }
51  }
52}
53
54// Swaps a keyed Image through its frames until it is unmounted or stopped. A still gets one
55// blit of its own frame, which sends nothing new but tells whether this terminal draws it.
56// Once the terminal refuses pictures no loop starts again: every blit would be refused too.
57export function animate(
58  $: EngineInterface,
59  { site, requestId, key, frames }: { site: Site; requestId: string; key: string; frames: Frame[] },
60  onDeny: (deny: string) => void,
61) {
62  const id = loopId(site, requestId, key)
63  const isStill = frames.length < 2
64  if (isTerminalRefusing || loops.has(id) || (isStill && checkedStills.has(id))) {
65    return
66  }
67  checkedStills.add(id)
68  const loop: Loop = { timer: undefined, isStopped: false }
69  loops.set(id, loop)
70  let index = 0
71  let hasLanded = false
72  let retries = 0
73  const schedule = (ms: number) => {
74    if (!loop.isStopped) {
75      loop.timer = $.clock.after(ms, tick)
76    }
77  }
78  const tick = async () => {
79    const next = (index + 1) % frames.length
80    const result = await $.ui.blit({ requestId, key, source: frames[next]!.source })
81    if (loop.isStopped) {
82      return
83    }
84    if (result.deny !== undefined) {
85      if (isTerminalDeny(result.deny)) {
86        isTerminalRefusing = true
87      } else if (!hasLanded && retries++ < RETRIES_BEFORE_MOUNT) {
88        schedule(RETRY_MS)
89        return
90      }
91      stop(id, loop)
92      onDeny(result.deny)
93      return
94    }
95    hasLanded = true
96    if (isStill) {
97      stop(id, loop)
98      return
99    }
100    index = next
101    schedule(frames[index]!.delayMs)
102  }
103  schedule(isStill ? 0 : frames[0]!.delayMs)
104}
105
106const reported = new Set<string>()
107
108// Logs the fix for a terminal deny once per session (ADR 0001).
109export function reportDeny($: EngineInterface, deny: string) {
110  const notice = noticeFor(deny)
111  if (notice === undefined || reported.has(notice)) {
112    return
113  }
114  reported.add(notice)
115  $.ui.log(notice)
116}
117

1import type { EngineInterface } from 'claude-code'
2
3import { isWindows, joinPath, osFrom, READ_BLOCK_BYTES, readChunkCommand, tmpRootFor } from '../domain/platform'
4import type { Host, Ports } from '../usecases/ports'
5
6const CACHE_DIRECTORY = 'media-view'
7const MIB = 1024 * 1024
8// $.fs.read rejects files over 4 MiB.
9const FS_READ_LIMIT_BYTES = 4 * MIB
10const MAX_FILE_BYTES = 64 * MIB
11// Base64 of a chunk, line breaks included, stays under a process's 4 MiB of output.
12const CHUNK_BYTES = 32 * READ_BLOCK_BYTES
13
14let host: Promise<Host> | undefined
15
16async function resolveHost($: EngineInterface): Promise<Host> {
17  const osVariable = await $.env.get('OS')
18  const os = isWindows(osVariable) ? 'windows' : osFrom({ osVariable, uname: (await $.process.run(['uname', '-s'])).stdout })
19  const tmpRoot = tmpRootFor(os, {
20    claudeTmpDir: await $.env.get('CLAUDE_CODE_TMPDIR'),
21    temp: os === 'windows' ? await $.env.get('TEMP') : undefined,
22    uid: os === 'windows' ? undefined : (await $.process.run(['id', '-u'])).stdout.trim(),
23  })
24  return { os, tmpRoot, cacheRoot: joinPath(os, tmpRoot, CACHE_DIRECTORY) }
25}
26
27// Resolved once per session; an attempt a render cut short is forgotten (ADR 0007).
28function sessionHost($: EngineInterface) {
29  host ??= resolveHost($).catch(error => {
30    host = undefined
31    throw error
32  })
33  return host
34}
35
36// Reads a file of any size up to MAX_FILE_BYTES: small ones whole, larger ones in base64 chunks (ADR 0005).
37async function readWhole($: EngineInterface, path: string) {
38  const { size } = await $.fs.stat(path)
39  if (size < FS_READ_LIMIT_BYTES) {
40    return Uint8Array.fromBase64((await $.fs.read(path, { as: 'bytes' })).base64)
41  }
42  if (size > MAX_FILE_BYTES) {
43    throw new Error(`${path} is ${Math.ceil(size / MIB)} MiB, over the ${MAX_FILE_BYTES / MIB} MiB media-view reads`)
44  }
45  const { os } = await sessionHost($)
46  const bytes = new Uint8Array(size)
47  for (let offset = 0; offset < size; offset += CHUNK_BYTES) {
48    const { argv, init } = readChunkCommand(os, path, offset, CHUNK_BYTES)
49    const read = await $.process.run(argv, init)
50    if (read.exitCode !== 0) {
51      throw new Error(`reading ${path} failed: ${read.stderr.trim()}`)
52    }
53    bytes.set(Uint8Array.fromBase64(read.stdout.replace(/\s+/g, '')).subarray(0, size - offset), offset)
54  }
55  return bytes
56}
57
58// The usecases' ports over this hook's `$` (ADR 0015).
59export function enginePorts($: EngineInterface): Ports {
60  return {
61    host: () => sessionHost($),
62    sessionId: () => $.session.id(),
63    readFile: path => readWhole($, path),
64    checkRead: path => $.tool.check({ tool: 'Read', input: { file_path: path } }),
65    // A Read call runs Read's permission path, dialog included; its result is not used.
66    askRead: async path => {
67      const asked = await $.tool.call({ tool: 'Read', file_path: path })
68      return asked.deny ?? (asked.isError ? asked.text : undefined)
69    },
70    listDirectory: async path => (await $.fs.list(path)).map(({ name, kind }) => ({ name, kind })),
71    exists: path => $.fs.exists(path),
72    run: async ({ argv, init }) => {
73      const ran = await $.process.run(argv, init)
74      if (ran.exitCode !== 0) {
75        throw new Error(`${argv[0]} exited ${ran.exitCode}: ${ran.stderr.trim()}`)
76      }
77    },
78    requestRedraw: afterMs => {
79      if (afterMs === undefined) {
80        $.ui.invalidate('ui.render')
81      } else {
82        $.clock.after(afterMs, () => $.ui.invalidate('ui.render'))
83      }
84    },
85  }
86}
87

1import type { EngineInterface, RenderElement, RenderInputOf } from 'claude-code'
2
3import { TILE_GAP, type AbovePromptTiles } from '../usecases/renders/abovePrompt'
4import { ENTRY_GAP, type MediaViewEntry } from '../usecases/renders/pane/mediaView'
5import { PICTURE_GAP, type PictureRows } from '../usecases/renders/pictureRows'
6import type { CellBox } from '../domain/layout'
7import { animate, reportDeny, stopAnimation, stopAnimations, stopAnimationsExcept, type Site } from './animator'
8
9const BUTTON_GAP = 1
10
11// An Image of another size is another Image: the terminal keeps an unchanged source at the size it was first sent (ADR 0017).
12const sizedKey = (key: string, { columns, rows }: CellBox) => `${key}@${columns}x${rows}`
13
14type TranscriptRow = RenderInputOf<'ToolGroup' | 'ToolResult' | 'UserMessage', 'terminal'>
15
16// The engine's own row, then the pictures under it; GIFs move only while the row is on screen (ADR 0008).
17export function drawPictureRows(
18  $: EngineInterface,
19  e: TranscriptRow,
20  { site, own, rows, isAnimated }: { site: Site; own: RenderElement; rows: PictureRows; isAnimated: boolean },
21) {
22  const { Box, Image } = $.ui.resolve(e)
23  const pictures = rows.flat().map(shown => ({ ...shown, key: sizedKey(shown.key, shown.box) }))
24  if (e.props.onScreen === null || !isAnimated) {
25    stopAnimations(site, e.requestId)
26  } else {
27    stopAnimationsExcept(site, e.requestId, pictures.map(({ key }) => key))
28    pictures.forEach(({ key, picture }) => animate($, { site, requestId: e.requestId, key, frames: picture.frames }, deny => reportDeny($, deny)))
29  }
30
31  return (
32    <Box flexDirection="column">
33      {own}
34      {rows.map(row => (
35        <Box flexDirection="row" gap={PICTURE_GAP}>
36          {row.map(({ key, picture, box }) => (
37            <Image key={sizedKey(key, box)} source={picture.frames[0]!.source} columns={box.columns} rows={box.rows} alt="image" />
38          ))}
39        </Box>
40      ))}
41    </Box>
42  )
43}
44
45// The draft's pictures above the prompt, each labelled `#n`, with `+N` for those left out.
46export function drawAbovePrompt($: EngineInterface, e: RenderInputOf<'AbovePrompt', 'terminal'>, { tiles, hidden }: AbovePromptTiles) {
47  const { Box, Image, Text } = $.ui.resolve(e)
48  const keys = tiles.map(({ n, box }) => sizedKey(`paste-${n}`, box))
49  stopAnimationsExcept('AbovePrompt', e.requestId, keys)
50  tiles.forEach(({ picture }, i) =>
51    animate($, { site: 'AbovePrompt', requestId: e.requestId, key: keys[i]!, frames: picture.frames }, deny => reportDeny($, deny)),
52  )
53
54  return (
55    <Box flexDirection="row" gap={TILE_GAP}>
56      {tiles.map(({ n, picture, box }) => (
57        <Box flexDirection="column">
58          <Image key={sizedKey(`paste-${n}`, box)} source={picture.frames[0]!.source} columns={box.columns} rows={box.rows} alt={`[Image #${n}]`} />
59          <Text dimColor>{`#${n}`}</Text>
60        </Box>
61      ))}
62      {hidden > 0 && <Text dimColor>{`+${hidden}`}</Text>}
63    </Box>
64  )
65}
66
67export type MediaViewButtons = {
68  copyPath: (path: string) => unknown
69  open: (path: string) => unknown
70  showInChat: (rowId: string) => unknown
71}
72
73// Each image of the session: its path and buttons above its picture, so text lines up whatever the width.
74export function drawMediaViewPane($: EngineInterface, e: RenderInputOf<'Pane', 'terminal'>, entries: MediaViewEntry[], buttons: MediaViewButtons) {
75  const { Box, Button, Image, Text } = $.ui.resolve(e)
76  if (entries.length === 0) {
77    return <Text dimColor>No images in this session yet.</Text>
78  }
79  // Only entries inside the pane's scroll window move (ADR 0008).
80  const keys = entries.map(({ id, box }) => sizedKey(id, box))
81  stopAnimationsExcept('Pane', e.requestId, keys)
82  entries.forEach(({ picture, isVisible }, i) =>
83    isVisible
84      ? animate($, { site: 'Pane', requestId: e.requestId, key: keys[i]!, frames: picture.frames }, deny => reportDeny($, deny))
85      : stopAnimation('Pane', e.requestId, keys[i]!),
86  )
87
88  return (
89    <Box flexDirection="column" gap={ENTRY_GAP}>
90      {entries.map(({ id, path, picture, box, rowId }) => (
91        <Box flexDirection="column">
92          <Text>{path}</Text>
93          <Box flexDirection="row" gap={BUTTON_GAP}>
94            <Button key={`copy-${id}`} label="Copy path" onPress={() => buttons.copyPath(path)} />
95            <Button key={`open-${id}`} label="Open" onPress={() => buttons.open(path)} />
96            {rowId !== undefined && <Button key={`show-${id}`} label="Show in chat" onPress={() => buttons.showInChat(rowId)} />}
97          </Box>
98          <Image key={sizedKey(id, box)} source={picture.frames[0]!.source} columns={box.columns} rows={box.rows} alt={path} />
99        </Box>
100      ))}
101    </Box>
102  )
103}
104

1export type Os = 'mac' | 'linux' | 'windows'
2
3// A process to run: its argument vector, and its stdin and extra environment.
4export type Command = { argv: string[]; init: { stdin?: string; env?: Record<string, string> } }
5
6const WINDOWS_OS_VARIABLE = 'Windows_NT'
7const MAC_UNAME = 'Darwin'
8const POSIX_TMP_BASE = '/tmp'
9// Paths reach child processes through the environment, never through a shell string.
10const PATH_VARIABLE = 'MEDIA_VIEW_PATH'
11const POWERSHELL = ['powershell.exe', '-NoProfile', '-NonInteractive', '-Command']
12
13export const osFrom = ({ osVariable, uname }: { osVariable: string | undefined; uname: string | undefined }): Os => {
14  if (osVariable === WINDOWS_OS_VARIABLE) {
15    return 'windows'
16  }
17  return uname?.trim() === MAC_UNAME ? 'mac' : 'linux'
18}
19
20export const isWindows = (osVariable: string | undefined) => osVariable === WINDOWS_OS_VARIABLE
21
22export const joinPath = (os: Os, ...parts: string[]) => parts.join(os === 'windows' ? '\\' : '/')
23
24// Where Claude Code keeps per-session files, the paste cache among them.
25export const tmpRootFor = (
26  os: Os,
27  { claudeTmpDir, temp, uid }: { claudeTmpDir: string | undefined; temp: string | undefined; uid: string | undefined },
28) => {
29  if (os === 'windows') {
30    return `${temp}\\claude`
31  }
32  const base = (claudeTmpDir ?? POSIX_TMP_BASE).replace(/\/+$/, '')
33  return `${base}/claude-${uid}`
34}
35
36// `$.fs.write` writes text only; binary goes through a process reading base64 on stdin.
37export const writeBytesCommand = (os: Os, path: string, bytes: Uint8Array): Command => {
38  const init = { stdin: bytes.toBase64(), env: { [PATH_VARIABLE]: path } }
39  if (os === 'windows') {
40    return {
41      argv: [...POWERSHELL, `[IO.File]::WriteAllBytes($env:${PATH_VARIABLE}, [Convert]::FromBase64String([Console]::In.ReadToEnd()))`],
42      init,
43    }
44  }
45  return { argv: ['/bin/sh', '-c', `base64 -d > "$${PATH_VARIABLE}"`], init }
46}
47
48export const makeDirectoryCommand = (os: Os, path: string): Command =>
49  os === 'windows'
50    ? { argv: [...POWERSHELL, `New-Item -ItemType Directory -Force -Path $env:${PATH_VARIABLE} | Out-Null`], init: { env: { [PATH_VARIABLE]: path } } }
51    : { argv: ['mkdir', '-p', path], init: {} }
52
53export const openCommand = (os: Os, path: string): Command =>
54  os === 'windows'
55    ? { argv: [...POWERSHELL, `Start-Process -FilePath $env:${PATH_VARIABLE}`], init: { env: { [PATH_VARIABLE]: path } } }
56    : os === 'mac'
57      ? { argv: ['open', path], init: {} }
58      : // xdg-open waits for some viewers to exit; in the background it returns at once.
59        { argv: ['/bin/sh', '-c', `xdg-open "$${PATH_VARIABLE}" >/dev/null 2>&1 &`], init: { env: { [PATH_VARIABLE]: path } } }
60
61const OFFSET_VARIABLE = 'MEDIA_VIEW_OFFSET'
62const LENGTH_VARIABLE = 'MEDIA_VIEW_LENGTH'
63// dd reads whole blocks; chunks are aligned to them.
64export const READ_BLOCK_BYTES = 65536
65
66// `$.fs.read` stops at 4 MiB and a process's output at 4 MiB too; a larger file is read as base64 slices.
67export const readChunkCommand = (os: Os, path: string, offset: number, length: number): Command => {
68  const env = { [PATH_VARIABLE]: path, [OFFSET_VARIABLE]: String(offset), [LENGTH_VARIABLE]: String(length) }
69  if (os === 'windows') {
70    return {
71      argv: [
72        ...POWERSHELL,
73        `$f = [IO.File]::OpenRead($env:${PATH_VARIABLE}); [void]$f.Seek([long]$env:${OFFSET_VARIABLE}, 0); $b = New-Object byte[] ([int]$env:${LENGTH_VARIABLE}); $n = $f.Read($b, 0, $b.Length); $f.Close(); [Convert]::ToBase64String($b, 0, $n)`,
74      ],
75      init: { env },
76    }
77  }
78  return {
79    argv: [
80      '/bin/sh',
81      '-c',
82      `dd if="$${PATH_VARIABLE}" bs=${READ_BLOCK_BYTES} skip=$((${OFFSET_VARIABLE} / ${READ_BLOCK_BYTES})) count=$((${LENGTH_VARIABLE} / ${READ_BLOCK_BYTES})) 2>/dev/null | base64`,
83    ],
84    init: { env },
85  }
86}
87

1import { MEDIA_VIEW_PANE } from '../renders/pane/mediaView'
2
3// Closing the media-view pane stops its GIFs; other panes are someone else's.
4export const mediaViewClosed = (paneId: string, actions: { stopPaneAnimations: () => void }) => {
5  if (paneId === MEDIA_VIEW_PANE) {
6    actions.stopPaneAnimations()
7  }
8}
9

1// What `$.command.register` declares for `/media-view`.
2export const MEDIA_VIEW_COMMAND = {
3  name: 'media-view',
4  description: 'Open the gallery of this session’s images; `still` or `animate` sets GIFs in the transcript',
5  argumentHint: '[still|animate]',
6}
7
8export type MediaViewActions = {
9  openPane: () => Promise<void>
10  setTranscriptAnimated: (isAnimated: boolean) => Promise<void>
11}
12
13// `/media-view` opens the media-view pane; `still` and `animate` set transcript GIFs. Answers the command's text.
14export const mediaView = async (actions: MediaViewActions, args: string) => {
15  const argument = args.trim()
16  if (argument === 'still' || argument === 'animate') {
17    const isAnimated = argument === 'animate'
18    await actions.setTranscriptAnimated(isAnimated)
19    return `Transcript GIFs: ${isAnimated ? 'animated' : 'still'}.`
20  }
21  if (argument !== '') {
22    return `Usage: /${MEDIA_VIEW_COMMAND.name} [still|animate]`
23  }
24  await actions.openPane()
25  return 'Opened the media gallery.'
26}
27

1import { imageFileName } from '../domain/imageTags'
2import { decodeMedia, isCompleteImage, isDecodedName } from '../domain/media/decode'
3import { cacheDirectoryName, framePlan, type Picture, type Plan } from '../domain/media/frameCache'
4import { joinPath, makeDirectoryCommand, writeBytesCommand } from '../domain/platform'
5import type { SourceImage } from '../domain/readOutput'
6import type { Host, Ports } from './ports'
7
8const IMAGES_DIRECTORY = 'images'
9const MAX_RETRIES = 3
10// Long enough for Claude Code to finish writing a paste the draft already shows.
11const RETRY_DELAY_MS = 250
12// Pictures are drawn at most 40 columns wide: frames past this edge, or past this count, only cost memory and disk (ADR 0006).
13const DRAWN_LIMITS = { maxEdge: 512, maxFrames: 200 }
14// Write processes running at once, across every picture (ADR 0007).
15const CONCURRENT_WRITES = 4
16
17// An image the transcript drew from a Read result, with the session and row that drew it.
18export type SeenImage = { key: string; path: string; picture: Promise<Picture | null>; sessionId: string; rowId: string }
19
20// A source not there yet (a paste Claude Code is still writing): a failure to retry, not "no picture".
21class NotWrittenYet extends Error {}
22
23const pictures = new Map<string, Promise<Picture | null>>()
24const plans = new Map<string, Plan>()
25const failures = new Map<string, number>()
26const seen = new Map<string, SeenImage>()
27const imageTagRows = new Map<string, string>()
28const readPaths = new Map<string, string>()
29const pasteCaches = new Map<string, string>()
30
31let writesRunning = 0
32const writesWaiting: (() => void)[] = []
33
34const withWriteSlot = async (write: () => Promise<void>) => {
35  if (writesRunning >= CONCURRENT_WRITES) {
36    await new Promise<void>(resolve => writesWaiting.push(resolve))
37  }
38  writesRunning++
39  try {
40    await write()
41  } finally {
42    writesRunning--
43    writesWaiting.shift()?.()
44  }
45}
46
47const decodeOrNull = (source: SourceImage | null) => {
48  try {
49    return source && decodeMedia(source.bytes, DRAWN_LIMITS)
50  } catch {
51    return null
52  }
53}
54
55// The picture's files, decoded once and kept until written. Undefined when the source is no picture or cannot be
56// decoded, which no retry changes; a load that throws (I/O, a paste not written yet) is the caller's to retry.
57const planFor = async (host: Host, key: string, load: () => Promise<SourceImage | null>) => {
58  const planned = plans.get(key)
59  if (planned !== undefined) {
60    return planned
61  }
62  const decoded = decodeOrNull(await load())
63  if (!decoded) {
64    return undefined
65  }
66  const plan = framePlan(host.os, joinPath(host.os, host.cacheRoot, await cacheDirectoryName(key)), decoded)
67  plans.set(key, plan)
68  return plan
69}
70
71const writePlan = async (ports: Ports, host: Host, { directory, writes, ...picture }: Plan): Promise<Picture> => {
72  await ports.run(makeDirectoryCommand(host.os, directory))
73  await Promise.all(writes.map(({ path, bytes }) => withWriteSlot(() => ports.run(writeBytesCommand(host.os, path, bytes)))))
74  return picture
75}
76
77// A failed load or write is forgotten and retried on a later draw, at most MAX_RETRIES times; a decoded plan is
78// kept, so a retry only writes. After that the picture stays absent (ADR 0007).
79const loadOrRetry = async (ports: Ports, key: string, load: () => Promise<SourceImage | null>): Promise<Picture | null> => {
80  try {
81    const host = await ports.host()
82    const plan = await planFor(host, key, load)
83    if (plan === undefined) {
84      return null
85    }
86    const picture = await writePlan(ports, host, plan)
87    plans.delete(key)
88    return picture
89  } catch {
90    const failed = (failures.get(key) ?? 0) + 1
91    failures.set(key, failed)
92    if (failed <= MAX_RETRIES) {
93      pictures.delete(key)
94      ports.requestRedraw(RETRY_DELAY_MS)
95    } else {
96      plans.delete(key)
97    }
98    return null
99  }
100}
101
102// Decodes and writes a source to the frame cache once per key; later calls share the result.
103// Resolves null, never rejects, when the source is no picture or every attempt failed.
104export const loadPicture = (ports: Ports, key: string, load: () => Promise<SourceImage | null>) => {
105  if (!pictures.has(key)) {
106    pictures.set(key, loadOrRetry(ports, key, load))
107  }
108  return pictures.get(key)!
109}
110
111// The paste cache folder of a session, `<tmp root>/<project>/<session>/images`, remembered once found.
112export const pasteCacheOf = async (ports: Ports, host: Host, sessionId: string) => {
113  const known = pasteCaches.get(sessionId)
114  if (known !== undefined) {
115    return known
116  }
117  for (const project of await ports.listDirectory(host.tmpRoot)) {
118    const images = joinPath(host.os, host.tmpRoot, project.name, sessionId, IMAGES_DIRECTORY)
119    if (project.kind === 'dir' && (await ports.exists(images))) {
120      pasteCaches.set(sessionId, images)
121      return images
122    }
123  }
124  return undefined
125}
126
127const readImageTag = async (ports: Ports, sessionId: string, n: number): Promise<SourceImage | null> => {
128  const host = await ports.host()
129  const images = await pasteCacheOf(ports, host, sessionId)
130  const name = images === undefined ? undefined : imageFileName((await ports.listDirectory(images)).map(entry => entry.name), n)
131  if (images === undefined || name === undefined) {
132    throw new NotWrittenYet(`[Image #${n}] is not in the paste cache yet`)
133  }
134  if (!isDecodedName(name)) {
135    return null
136  }
137  const bytes = await ports.readFile(joinPath(host.os, images, name))
138  if (!isCompleteImage(bytes)) {
139    throw new NotWrittenYet(`[Image #${n}] is still being written`)
140  }
141  return { bytes }
142}
143
144// The picture of `[Image #n]`, from the session's paste cache (ADR 0004).
145export const loadImageTag = (ports: Ports, sessionId: string, n: number) =>
146  loadPicture(ports, `paste/${sessionId}/${n}`, () => readImageTag(ports, sessionId, n))
147
148export const rememberSeen = (image: SeenImage) => {
149  seen.set(image.key, image)
150}
151
152// The images the transcript drew from Read results in a session, oldest first.
153export const seenImages = (sessionId: string) => [...seen.values()].filter(image => image.sessionId === sessionId)
154
155export const rememberImageTagRow = (sessionId: string, n: number, rowId: string) => {
156  imageTagRows.set(`${sessionId}/${n}`, rowId)
157}
158
159// The transcript row that drew `[Image #n]`, if one has.
160export const imageTagRowOf = (sessionId: string, n: number) => imageTagRows.get(`${sessionId}/${n}`)
161
162// The file a Read call read, kept from its `tool.call` for rows whose props do not carry it.
163export const rememberReadPath = (toolUseId: string, path: string) => {
164  readPaths.set(toolUseId, path)
165}
166
167export const readPathOf = (toolUseId: string) => readPaths.get(toolUseId)
168

1import { imageTagNumbers } from '../../domain/imageTags'
2import { cellBox, type CellBox } from '../../domain/layout'
3import type { Picture } from '../../domain/media/frameCache'
4import { loadImageTag } from '../images'
5import type { Ports } from '../ports'
6
7// Pasting raises no edit event, so the draft is read on a timer.
8const DRAFT_POLL_MS = 200
9const MAX_TILE_COLUMNS = 24
10const MAX_TILE_ROWS = 6
11const LABEL_ROWS = 1
12const MORE_LABEL_COLUMNS = 4
13export const TILE_GAP = 1
14
15export type Tile = { n: number; picture: Picture; box: CellBox }
16
17// What the `AbovePrompt` band draws: the tiles that fit, and how many are left out.
18export type AbovePromptTiles = { tiles: Tile[]; hidden: number }
19
20export type DraftPorts = {
21  every: (ms: number, tick: () => Promise<void>) => { cancel: () => void }
22  readDraft: () => Promise<string>
23  updateDraftNumbers: (change: (current: readonly number[]) => readonly number[]) => Promise<void>
24}
25
26const isSameList = (a: readonly number[], b: readonly number[]) => a.length === b.length && a.every((n, i) => n === b[i])
27
28let watch: { cancel: () => void } | undefined
29
30// Keeps the draft's image tag numbers up to date, written only when they change.
31export const watchDraft = (draft: DraftPorts) => {
32  let lastDraft: string | undefined
33  let isReading = false
34  watch?.cancel()
35  watch = draft.every(DRAFT_POLL_MS, async () => {
36    if (isReading) {
37      return
38    }
39    isReading = true
40    try {
41      const text = await draft.readDraft()
42      if (text !== lastDraft) {
43        lastDraft = text
44        const numbers = imageTagNumbers(text)
45        await draft.updateDraftNumbers(current => (isSameList(current, numbers) ? current : numbers))
46      }
47    } finally {
48      isReading = false
49    }
50  })
51}
52
53// The tiles that fit in `columns`, keeping room for a `+N` label when some are left out.
54const fitTiles = (tiles: Tile[], columns: number) => {
55  const fitted: Tile[] = []
56  let used = 0
57  for (const tile of tiles) {
58    const needed = tile.box.columns + (fitted.length > 0 ? TILE_GAP : 0)
59    const reserve = fitted.length + 1 < tiles.length ? MORE_LABEL_COLUMNS : 0
60    if (used + needed + reserve > columns) {
61      break
62    }
63    fitted.push(tile)
64    used += needed
65  }
66  return fitted
67}
68
69// The pictures of the draft's image tags, one row above the prompt; null when there are none.
70export const abovePrompt = async (
71  ports: Ports,
72  { numbers, bodyColumns, maxRows }: { numbers: readonly number[]; bodyColumns: number; maxRows: number },
73): Promise<AbovePromptTiles | null> => {
74  if (numbers.length === 0) {
75    return null
76  }
77  const sessionId = await ports.sessionId()
78  const tileRows = Math.max(1, Math.min(MAX_TILE_ROWS, maxRows - LABEL_ROWS))
79  const loaded = await Promise.all(numbers.map(n => loadImageTag(ports, sessionId, n).then(picture => picture && { n, picture })))
80  const tiles = loaded
81    .filter((item): item is { n: number; picture: Picture } => item !== null)
82    .map(item => ({ ...item, box: cellBox(item.picture, { maxColumns: MAX_TILE_COLUMNS, maxRows: tileRows }) }))
83  if (tiles.length === 0) {
84    return null
85  }
86  const shown = fitTiles(tiles, bodyColumns)
87  return { tiles: shown, hidden: tiles.length - shown.length }
88}
89

1import { cellBox, type CellBox } from '../../../domain/layout'
2import { mediaTypeOf } from '../../../domain/media/decode'
3import type { Picture } from '../../../domain/media/frameCache'
4import { joinPath } from '../../../domain/platform'
5import { imageTagRowOf, loadImageTag, pasteCacheOf, seenImages } from '../../images'
6import type { Ports } from '../../ports'
7
8export const MEDIA_VIEW_PANE = 'media-view'
9const MAX_COLUMNS = 24
10const MAX_ROWS = 8
11
12// Rows between entries, and the row each entry's buttons take.
13export const ENTRY_GAP = 1
14const BUTTON_ROWS = 1
15
16// One image of the media-view pane: `rowId` is the transcript row to show it in, when one drew it;
17// `isVisible` says whether any of its rows is inside the pane's scroll window.
18export type MediaViewEntry = { id: string; path: string; picture: Picture; box: CellBox; rowId: string | undefined; isVisible: boolean }
19
20export type PaneScroll = { offset: number; bodyRows: number }
21
22// The paste cache, when the tmp root can be listed; a gallery without it still lists Read images.
23const listPasteCache = async (ports: Ports, sessionId: string) => {
24  const host = await ports.host()
25  try {
26    const images = await pasteCacheOf(ports, host, sessionId)
27    return images === undefined ? undefined : { host, images, files: await ports.listDirectory(images) }
28  } catch {
29    return undefined
30  }
31}
32
33const pastedImages = async (ports: Ports, sessionId: string) => {
34  const pasteCache = await listPasteCache(ports, sessionId)
35  if (pasteCache === undefined) {
36    return []
37  }
38  const { host, images, files } = pasteCache
39  const numbered = files
40    .filter(file => file.kind === 'file' && mediaTypeOf(file.name) !== undefined)
41    .map(file => ({ n: Number.parseInt(file.name, 10), path: joinPath(host.os, images, file.name) }))
42    .filter(({ n }) => Number.isInteger(n))
43    .sort((a, b) => b.n - a.n)
44  return Promise.all(
45    numbered.map(async ({ n, path }) => ({
46      id: `paste-${n}`,
47      path,
48      picture: await loadImageTag(ports, sessionId, n),
49      rowId: imageTagRowOf(sessionId, n),
50    })),
51  )
52}
53
54const readImages = (sessionId: string) =>
55  Promise.all(
56    seenImages(sessionId)
57      .reverse()
58      .map(async ({ key, path, picture, rowId }) => ({ id: `read-${key}`, path, picture: await picture, rowId })),
59  )
60
61// Every image of the session: pasted ones newest first, then the ones Read results drew, newest first.
62export const mediaViewPane = async (
63  ports: Ports,
64  { bodyColumns, scroll }: { bodyColumns: number; scroll: PaneScroll },
65): Promise<MediaViewEntry[]> => {
66  const sessionId = await ports.sessionId()
67  const shown = [...(await pastedImages(ports, sessionId)), ...(await readImages(sessionId))].flatMap(entry =>
68    entry.picture === null
69      ? []
70      : [{ ...entry, picture: entry.picture, box: cellBox(entry.picture, { maxColumns: Math.min(MAX_COLUMNS, bodyColumns), maxRows: MAX_ROWS }) }],
71  )
72  let top = 0
73  return shown.map(entry => {
74    const rows = Math.ceil(entry.path.length / bodyColumns) + BUTTON_ROWS + entry.box.rows
75    const isVisible = top < scroll.offset + scroll.bodyRows && top + rows > scroll.offset
76    top += rows + ENTRY_GAP
77    return { ...entry, isVisible }
78  })
79}
80

1import { imageFromReadOutput, isImageReadOutput } from '../../domain/readOutput'
2import { loadPicture, readPathOf, rememberSeen } from '../images'
3import type { Ports } from '../ports'
4import { isShown, placeInRows, type PictureRows, type Shown } from './pictureRows'
5
6export type ToolGroupCall = { tool_use_id?: string; tool: string; input: unknown; output?: unknown }
7
8// The pictures of the Read calls folded into one `ToolGroup` row; null when it has none.
9export const toolGroup = async (
10  ports: Ports,
11  { requestId, calls, viewportColumns }: { requestId: string; calls: readonly ToolGroupCall[]; viewportColumns: number | undefined },
12): Promise<PictureRows | null> => {
13  const images = calls.flatMap((call, i) => (call.tool === 'Read' && isImageReadOutput(call.output) ? [{ call, i }] : []))
14  if (images.length === 0) {
15    return null
16  }
17  const sessionId = await ports.sessionId()
18  const loading = images.map(({ call, i }) => {
19    const key = call.tool_use_id ?? `${requestId}/${i}`
20    // The base64 is decoded only when the picture is not cached yet.
21    const picture = loadPicture(ports, key, async () => imageFromReadOutput(call.output))
22    const inputPath = (call.input as { file_path?: unknown } | undefined)?.file_path
23    const path = typeof inputPath === 'string' ? inputPath : readPathOf(key)
24    if (path !== undefined) {
25      rememberSeen({ key, path, picture, sessionId, rowId: requestId })
26    }
27    return picture.then(loaded => loaded && { key: `read-${i}`, picture: loaded })
28  })
29  const shown: Shown[] = (await Promise.all(loading)).filter(isShown)
30  return shown.length === 0 ? null : placeInRows(shown, viewportColumns)
31}
32

1import { imageFromReadOutput, isImageReadOutput } from '../../domain/readOutput'
2import { loadPicture, readPathOf, rememberSeen } from '../images'
3import type { Ports } from '../ports'
4import { placeInRows, type PictureRows } from './pictureRows'
5
6// The picture of a Read shown on its own `ToolResult` row; null for any other result.
7export const toolResult = async (
8  ports: Ports,
9  { toolUseId, tool, output, viewportColumns }: { toolUseId: string; tool: string; output: unknown; viewportColumns: number | undefined },
10): Promise<PictureRows | null> => {
11  if (tool !== 'Read' || !isImageReadOutput(output)) {
12    return null
13  }
14  const picture = loadPicture(ports, toolUseId, async () => imageFromReadOutput(output))
15  const path = readPathOf(toolUseId)
16  if (path !== undefined) {
17    rememberSeen({ key: toolUseId, path, picture, sessionId: await ports.sessionId(), rowId: toolUseId })
18  }
19  const loaded = await picture
20  return loaded ? placeInRows([{ key: 'read', picture: loaded }], viewportColumns) : null
21}
22