image-peek

1// The paste detection and the cache lookup are adapted from jarrodwatts/claude-image-view (hooks/register.tsx),
2// MIT License, Copyright (c) 2026 Jarrod Watts. Its notice is in this plugin's LICENSE.
3import { atom, read, update } from "claude-code";
4import type { EngineInterface, Register } from "claude-code";
5
6import type { PastedImage } from "../types";
7import { fitRow, imageNumbers, pngSize } from "./layout";
8import type { Cells, Size } from "./layout";
9import { fromBase64, quadrantCells, parseBmp } from "./raster";
10import { drawingFor } from "./terminal";
11import type { Drawing } from "./terminal";
12
13// Pasting an image raises no prompt.edit (the tag only shows up on the next keystroke), so the draft is polled.
14const POLL_MS = 200;
15
16const imagesAtom = atom({ plugin: "image-peek", key: "images" } as const, [] as PastedImage[]);
17
18type Engine = EngineInterface;
19
20let tmpRoot: string | undefined;
21let found: { sessionId: string; dir: string } | undefined;
22let drawing: Drawing | undefined;
23// The image numbers last drawn, so an unchanged draft doesn't rewrite state; undefined while a drawn
24// image's file is still missing, so the next poll looks again.
25let shownKey: string | undefined;
26let isChecking = false;
27const sizes = new Map<string, Size | null>();
28const thumbnails = new Map<string, Promise<string | null>>();
29
30// Claude Code caches each paste as <tmp>/<project>/<session>/images/<n>.png. The project folder is named
31// after a working directory that may since have moved, so it is found by the session id instead.
32const imagesDir = async ($: Engine): Promise<string | undefined> => {
33    const sessionId = await $.session.id();
34    if (found?.sessionId === sessionId) {
35        return found.dir;
36    }
37    if (tmpRoot === undefined) {
38        const fromEnv = await $.env.get("CLAUDE_CODE_TMPDIR");
39        tmpRoot = fromEnv ?? `/tmp/claude-${(await $.process.run(["id", "-u"])).stdout.trim()}`;
40    }
41    const entries = await $.fs.list(tmpRoot).catch(() => []);
42    for (const entry of entries) {
43        const dir = `${tmpRoot}/${entry.name}/${sessionId}/images`;
44        if (entry.kind === "dir" && (await $.fs.exists(dir))) {
45            found = { sessionId, dir };
46            return dir;
47        }
48    }
49    return undefined;
50};
51
52const describe = async ($: Engine, dir: string | undefined, n: number): Promise<PastedImage> => {
53    const path = `${dir}/${n}.png`;
54    if (dir === undefined || !(await $.fs.exists(path))) {
55        return { n, path: null, size: null };
56    }
57    if (!sizes.has(path)) {
58        const head = await $.fs.read(path, { as: "bytes" }).then(
59            ({ base64 }) => pngSize(base64),
60            // Too big to read: still drawable, just without its aspect ratio.
61            () => undefined,
62        );
63        if (head === null) {
64            return { n, path: null, size: null };
65        }
66        sizes.set(path, head ?? null);
67    }
68    return { n, path, size: sizes.get(path) ?? null };
69};
70
71const show = async ($: Engine, draft: string) => {
72    const numbers = imageNumbers(draft);
73    const key = numbers.join(",");
74    if (key === shownKey) {
75        return;
76    }
77    const dir = numbers.length > 0 ? await imagesDir($) : undefined;
78    const list: PastedImage[] = [];
79    for (const n of numbers) {
80        list.push(await describe($, dir, n));
81    }
82    shownKey = list.every((image) => image.path !== null) ? key : undefined;
83    await update($, imagesAtom, () => list);
84};
85
86const check = async ($: Engine) => {
87    if (isChecking) {
88        return;
89    }
90    isChecking = true;
91    try {
92        await show($, (await $.prompt.read()).text);
93    } finally {
94        isChecking = false;
95    }
96};
97
98const drawingHere = async ($: Engine): Promise<Drawing> => {
99    if (drawing === undefined) {
100        drawing = drawingFor({
101            forceImages: await $.env.get("CLAUDE_CODE_FORCE_TERMINAL_IMAGES"),
102            tmux: await $.env.get("TMUX"),
103            termProgram: await $.env.get("TERM_PROGRAM"),
104            term: await $.env.get("TERM"),
105            kittyWindowId: await $.env.get("KITTY_WINDOW_ID"),
106        });
107    }
108    return drawing;
109};
110
111// sips scales the PNG to exactly one pixel per quarter cell and writes it as a BMP, which is cheap to read here.
112const renderThumbnail = async ($: Engine, path: string, { columns, rows }: Cells): Promise<string | null> => {
113    const bmp = path.replace(/\.png$/, `.image-peek-${columns}x${rows}.bmp`);
114    const result = await $.process.run([
115        "sips",
116        "-z",
117        String(rows * 2),
118        String(columns * 2),
119        "-s",
120        "format",
121        "bmp",
122        path,
123        "--out",
124        bmp,
125    ]);
126    if (result.exitCode !== 0) {
127        throw new Error(`sips exited ${result.exitCode}: ${result.stderr}`);
128    }
129    const { base64 } = await $.fs.read(bmp, { as: "bytes" });
130    const pixels = parseBmp(fromBase64(base64));
131    return pixels === null ? null : quadrantCells(pixels, columns, rows);
132};
133
134const thumbnail = ($: Engine, path: string, cells: Cells): Promise<string | null> => {
135    const key = `${path}@${cells.columns}x${cells.rows}`;
136    let pending = thumbnails.get(key);
137    if (pending === undefined) {
138        // A failed thumbnail is forgotten, so the next drawing tries again.
139        pending = renderThumbnail($, path, cells).catch(() => {
140            thumbnails.delete(key);
141            return null;
142        });
143        thumbnails.set(key, pending);
144    }
145    return pending;
146};
147
148// Quick Look stays open until the person closes it, longer than a process call may last, so it is detached; started
149// from a background process its panel opens behind the terminal, so it is then brought to the front.
150const QUICK_LOOK = `qlmanage -p "$1" >/dev/null 2>&1 &
151sleep 0.5
152osascript -e "tell application \\"System Events\\" to set frontmost of (first process whose unix id is $!) to true" >/dev/null 2>&1`;
153
154const enlarge = ($: Engine, path: string) => {
155    void $.process.run(["sh", "-c", QUICK_LOOK, "image-peek", path]);
156};
157
158export const register: Register = (on) => {
159    on("session.start", async ($, e, next) => {
160        $.clock.every(POLL_MS, () => check($));
161        return next(e);
162    });
163
164    on("ui.render", { component: "AbovePrompt" }, async ($, e, next) => {
165        if (e.surface !== "terminal" || e.props.hasSurvey) {
166            return next(e);
167        }
168        const list = await read($, imagesAtom);
169        if (list.length === 0) {
170            return next(e);
171        }
172
173        const { Box, Button, Image, Raster, Text } = $.ui.resolve(e);
174        const mode = await drawingHere($);
175        const cells = fitRow(
176            list.map((image) => image.size),
177            e.props.maxRows,
178            e.props.bodyColumns,
179        );
180        const pictures = await Promise.all(
181            list.map((image, i) =>
182                image.path === null || mode === "pixels"
183                    ? Promise.resolve(null)
184                    : thumbnail($, image.path, cells[i] ?? { columns: 4, rows: 1 }),
185            ),
186        );
187        const below = await next(e);
188
189        return (
190            <Box flexDirection="column">
191                <Box flexDirection="row" columnGap={1}>
192                    {list.map((image, i) => {
193                        const { columns, rows } = cells[i] ?? { columns: 4, rows: 1 };
194                        const path = image.path;
195                        const picture = pictures[i];
196                        return (
197                            <Box flexDirection="column" alignItems="center" borderStyle="round" borderDimColor>
198                                {path === null ? (
199                                    <Box width={columns} height={rows} alignItems="center" justifyContent="center">
200                                        <Text dimColor wrap="truncate">
201                                            no preview
202                                        </Text>
203                                    </Box>
204                                ) : mode === "pixels" ? (
205                                    <Image
206                                        key={`image-${image.n}`}
207                                        source={{ file: path, format: "png" }}
208                                        columns={columns}
209                                        rows={rows}
210                                        alt={`[Image #${image.n}]`}
211                                    />
212                                ) : picture === null ? (
213                                    <Box width={columns} height={rows} alignItems="center" justifyContent="center">
214                                        <Text dimColor wrap="truncate">
215                                            [Image #{image.n}]
216                                        </Text>
217                                    </Box>
218                                ) : (
219                                    <Raster key={`image-${image.n}`} columns={columns} rows={rows} cells={picture} />
220                                )}
221                                {path === null ? (
222                                    <Text dimColor>#{image.n}</Text>
223                                ) : (
224                                    <Button
225                                        key={`enlarge-${image.n}`}
226                                        label={image.n <= 9 ? "⤢" : `⤢ #${image.n}`}
227                                        hotkey={image.n <= 9 ? String(image.n) : undefined}
228                                        plain
229                                        dimColor
230                                        onPress={() => enlarge($, path)}
231                                    />
232                                )}
233                            </Box>
234                        );
235                    })}
236                </Box>
237                {below}
238            </Box>
239        );
240    });
241};
242

1// Adapted from jarrodwatts/claude-image-view (hooks/layout.ts), MIT License, Copyright (c) 2026 Jarrod Watts.
2// Its notice is in this plugin's LICENSE.
3
4export type Size = { width: number; height: number };
5export type Cells = { columns: number; rows: number };
6
7const TILE_ROWS = 12;
8const MAX_COLUMNS = 48;
9const MIN_COLUMNS = 4;
10// A terminal cell is about twice as tall as it is wide.
11const CELL_ASPECT = 2;
12// Used when the size is unknown (a file over $.fs.read's 4 MiB cap, or no file).
13const FALLBACK: Size = { width: 16, height: 10 };
14// Each tile adds a border on every side and the enlarge button under the picture.
15const TILE_CHROME_ROWS = 3;
16const TILE_CHROME_COLUMNS = 2;
17const GAP = 1;
18
19/** The distinct image numbers a draft references, in the order they first appear. */
20export const imageNumbers = (draft: string): number[] => {
21    const seen = new Set<number>();
22    for (const match of draft.matchAll(/\[Image #(\d+)\]/g)) {
23        seen.add(Number(match[1]));
24    }
25    return [...seen];
26};
27
28/** Width and height from a PNG's IHDR chunk, or null when the bytes aren't a PNG. */
29export const pngSize = (base64: string): Size | null => {
30    // 24 bytes cover the signature and IHDR's width and height; 32 base64 chars decode to exactly 24.
31    const head = Uint8Array.from(atob(base64.slice(0, 32)), (char) => char.charCodeAt(0));
32    const signature = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
33    if (head.length < 24 || signature.some((byte, i) => head[i] !== byte)) {
34        return null;
35    }
36    const view = new DataView(head.buffer, head.byteOffset, head.byteLength);
37    const width = view.getUint32(16);
38    const height = view.getUint32(20);
39    return width > 0 && height > 0 ? { width, height } : null;
40};
41
42/** A picture box `rows` tall that keeps the picture's aspect ratio. */
43export const fitCells = (size: Size | null, tileRows = TILE_ROWS): Cells => {
44    const { width, height } = size ?? FALLBACK;
45    let rows = tileRows;
46    let columns = Math.round((rows * CELL_ASPECT * width) / height);
47    if (columns > MAX_COLUMNS) {
48        columns = MAX_COLUMNS;
49        rows = Math.max(1, Math.round((MAX_COLUMNS * height) / (CELL_ASPECT * width)));
50    }
51    return { columns: Math.max(MIN_COLUMNS, columns), rows: Math.min(rows, tileRows) };
52};
53
54/**
55 * Picture boxes for one row of tiles that fits the band whole, so it never scrolls:
56 * the tallest tiles whose chrome fits in `maxRows` and whose total width fits in `bodyColumns`.
57 */
58export const fitRow = (sizes: readonly (Size | null)[], maxRows: number, bodyColumns: number): Cells[] => {
59    const tallest = Math.max(1, Math.min(TILE_ROWS, maxRows - TILE_CHROME_ROWS));
60    for (let tileRows = tallest; tileRows > 1; tileRows--) {
61        const cells = sizes.map((size) => fitCells(size, tileRows));
62        const width = cells.reduce((sum, c) => sum + c.columns + TILE_CHROME_COLUMNS, 0) + GAP * (cells.length - 1);
63        if (width <= bodyColumns) {
64            return cells;
65        }
66    }
67    return sizes.map((size) => fitCells(size, 1));
68};
69

1export type Pixels = { width: number; height: number; rgb: Uint8Array };
2
3const FULL_BLOCK = 0x2588;
4// The quadrant glyph for each mask of foreground pixels: 1 top left, 2 top right, 4 bottom left, 8 bottom right.
5// Mask 0 never happens, since the first seed is always foreground.
6const QUADRANTS = [
7    0x20, 0x2598, 0x259d, 0x2580, 0x2596, 0x258c, 0x259e, 0x259b, 0x2597, 0x259a, 0x2590, 0x259c, 0x2584, 0x2599,
8    0x259f, 0x2588,
9];
10
11/**
12 * The pixels of an uncompressed 24- or 32-bit BMP (what `sips -s format bmp` writes), top row first;
13 * null for anything else.
14 */
15export const parseBmp = (bytes: Uint8Array): Pixels | null => {
16    if (bytes.length < 54 || bytes[0] !== 0x42 || bytes[1] !== 0x4d) {
17        return null;
18    }
19    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
20    const offset = view.getUint32(10, true);
21    const width = view.getInt32(18, true);
22    const signedHeight = view.getInt32(22, true);
23    const bitsPerPixel = view.getUint16(28, true);
24    const compression = view.getUint32(30, true);
25    const isBitfields = compression === 3 && bitsPerPixel === 32;
26    if (width <= 0 || signedHeight === 0 || (bitsPerPixel !== 24 && bitsPerPixel !== 32)) {
27        return null;
28    }
29    if (compression !== 0 && !isBitfields) {
30        return null;
31    }
32    const height = Math.abs(signedHeight);
33    const isBottomUp = signedHeight > 0;
34    const bytesPerPixel = bitsPerPixel / 8;
35    const stride = Math.ceil((width * bytesPerPixel) / 4) * 4;
36    if (offset + stride * height > bytes.length) {
37        return null;
38    }
39    const rgb = new Uint8Array(width * height * 3);
40    for (let y = 0; y < height; y++) {
41        const row = offset + stride * (isBottomUp ? height - 1 - y : y);
42        for (let x = 0; x < width; x++) {
43            const source = row + x * bytesPerPixel;
44            const target = (y * width + x) * 3;
45            rgb[target] = bytes[source + 2];
46            rgb[target + 1] = bytes[source + 1];
47            rgb[target + 2] = bytes[source];
48        }
49    }
50    return { width, height, rgb };
51};
52
53type Rgb = [number, number, number];
54
55const pixelAt = (pixels: Pixels, x: number, y: number): Rgb => {
56    const index = (y * pixels.width + x) * 3;
57    return [pixels.rgb[index], pixels.rgb[index + 1], pixels.rgb[index + 2]];
58};
59
60const distance = (a: Rgb, b: Rgb): number => (a[0] - b[0]) ** 2 + (a[1] - b[1]) ** 2 + (a[2] - b[2]) ** 2;
61
62const average = (colors: Rgb[]): number => {
63    const channel = (i: number) => Math.round(colors.reduce((sum, color) => sum + color[i], 0) / colors.length);
64    return (channel(0) << 16) | (channel(1) << 8) | channel(2);
65};
66
67/**
68 * One cell for its 2x2 pixels (top left, top right, bottom left, bottom right): the two most different pixels
69 * seed a foreground and a background, every pixel joins the nearer, and the glyph covers the foreground ones.
70 */
71const quadrantCell = (quad: Rgb[]): [number, number, number] => {
72    let seeds: [number, number] = [0, 0];
73    let widest = -1;
74    for (let a = 0; a < 4; a++) {
75        for (let b = a + 1; b < 4; b++) {
76            const d = distance(quad[a], quad[b]);
77            if (d > widest) {
78                widest = d;
79                seeds = [a, b];
80            }
81        }
82    }
83    if (widest === 0) {
84        const color = average(quad);
85        return [FULL_BLOCK, color, color];
86    }
87    let mask = 0;
88    const foreground: Rgb[] = [];
89    const background: Rgb[] = [];
90    quad.forEach((pixel, i) => {
91        if (distance(pixel, quad[seeds[0]]) <= distance(pixel, quad[seeds[1]])) {
92            mask |= 1 << i;
93            foreground.push(pixel);
94        } else {
95            background.push(pixel);
96        }
97    });
98    return [QUADRANTS[mask], average(foreground), average(background)];
99};
100
101/**
102 * A Raster's `cells` for pixels exactly `columns * 2` wide and `rows * 2` tall: each cell a quadrant block
103 * drawing its four pixels in the two colors that fit them best.
104 */
105export const quadrantCells = (pixels: Pixels, columns: number, rows: number): string | null => {
106    if (pixels.width !== columns * 2 || pixels.height !== rows * 2) {
107        return null;
108    }
109    const words = new Uint32Array(columns * rows * 3);
110    for (let row = 0; row < rows; row++) {
111        for (let column = 0; column < columns; column++) {
112            const x = column * 2;
113            const y = row * 2;
114            const quad = [
115                pixelAt(pixels, x, y),
116                pixelAt(pixels, x + 1, y),
117                pixelAt(pixels, x, y + 1),
118                pixelAt(pixels, x + 1, y + 1),
119            ];
120            words.set(quadrantCell(quad), (row * columns + column) * 3);
121        }
122    }
123    return toBase64(new Uint8Array(words.buffer));
124};
125
126const toBase64 = (bytes: Uint8Array): string => {
127    let binary = "";
128    for (let start = 0; start < bytes.length; start += 0x8000) {
129        binary += String.fromCharCode(...bytes.subarray(start, start + 0x8000));
130    }
131    return btoa(binary);
132};
133
134export const fromBase64 = (base64: string): Uint8Array => Uint8Array.from(atob(base64), (char) => char.charCodeAt(0));
135

1export type TerminalEnv = {
2    forceImages?: string;
3    tmux?: string;
4    termProgram?: string;
5    term?: string;
6    kittyWindowId?: string;
7};
8
9export type Drawing = "pixels" | "blocks";
10
11/**
12 * Whether Claude Code's `Image` can show real pixels here. It speaks only kitty graphics with Unicode
13 * placeholders, which kitty and Ghostty draw and Warp, tmux and the rest do not; everywhere else the
14 * picture is drawn with quadrant block characters instead.
15 */
16export const drawingFor = (env: TerminalEnv): Drawing => {
17    if (env.forceImages === "1" || env.forceImages === "true") {
18        return "pixels";
19    }
20    if (env.tmux !== undefined && env.tmux !== "") {
21        return "blocks";
22    }
23    const program = env.termProgram?.toLowerCase();
24    if (program === "ghostty" || program === "kitty") {
25        return "pixels";
26    }
27    if (env.kittyWindowId !== undefined || env.term === "xterm-kitty" || env.term === "xterm-ghostty") {
28        return "pixels";
29    }
30    return "blocks";
31};
32

1export type PastedImage = {
2    n: number;
3    /** Absolute path of the cached PNG; null when it can't be found. */
4    path: string | null;
5    /** Pixel size; null when unknown, and the tile falls back to a default shape. */
6    size: { width: number; height: number } | null;
7};
8
9declare module "claude-code" {
10    interface PluginState {
11        "image-peek": { images: PastedImage[] };
12    }
13}
14