image-preview

1// image-preview — render image paths inline, like the desktop GUI app.
2//
3// Two render paths, keyed on the same scan of text:
4//   1. UserMessage rows: an image reference in the prompt draws a thumbnail
5//      below the prompt's own block, aspect-correct, never taller than 10 rows.
6//   2. AssistantMessage rows: an image path the model mentions draws the same
7//      thumbnail in the reply row.
8//
9// Both hooks keep the engine's own block verbatim via next(e) and only append,
10// so a prompt with a picture renders exactly like one without.
11//
12// An image reference is a path ending in a supported extension (.png .jpg
13// .jpeg .gif .webp .bmp .tif .tiff .heic .heif .ico .svg), optionally with a
14// ?query/#fragment, or an [Image #N] token for a picture pasted into the
15// prompt, read back from the transcript. Each picture is decoded once (sips ->
16// a bounded PNG the terminal reads, as the mermaid pane hands its diagrams
17// over) and memoised by path, or by content for a paste.
18const NS = { plugin: "image-preview", key: "state" };
19const TMP_DIR = "/tmp/image-preview";
20
21// A terminal cell is ~2.125x taller than wide (16px x 34px); a block whose cell
22// aspect matches the picture's pixel aspect shows it undistorted.
23const CELL_H_OVER_CELL_W = 34 / 16;
24const MAX_ROWS = 10;
25
26// A whitespace-delimited token ending in a supported image extension. The
27// capture is the bare path: an optional ?query/#fragment suffix (as in
28// /a/pic.png?w=100) is matched but left out of group 1, so the file the
29// decoder reads stays clean. The path may sit in a code span, bold, or a
30// markdown link target, and may be followed by sentence punctuation; a path
31// never starts with the [, ( or * of that markup nor holds a link's "](", so
32// `[a](/b.png)` reads /b.png while app/[id]/hero.png stays whole. A
33// backslash-escaped space (as a file dropped on the terminal arrives) is part
34// of the path, never the start of one.
35const PATH_RE =
36  /(?:^|(?<!\\)[\s"'`([,{*])([^\s"'<>`*([\]](?:\\ |(?!\]\()[^\s"'<>`])*?\.(?:png|jpg|jpeg|gif|webp|bmp|tif(?:f)?|heic|heif|ico|svg))(?:\?[^#\s"')]*)?(?:#[^\s"')]*)?(?=[\s"'`*)\]}]|[.,:;!?](?:[\s"'`*)\]}]|$)|$)/gi;
37// A quoted path with a space in it ("…", '…' or a code span) starting at /,
38// ~/, ./ or ../; group 2 is the path. Unquoted, a space ends a path.
39const QUOTED_RE =
40  /(["'`])((?:\/|~\/|\.\.?\/)(?=[^"'`\n<>]* )[^"'`\n<>]*?\.(?:png|jpg|jpeg|gif|webp|bmp|tif(?:f)?|heic|heif|ico|svg))\1/gi;
41// An [Image #N] token the engine puts in a prompt for a pasted picture; group
42// 1 is its number, counted across the session.
43const REF_RE = /\[Image\s*#(\d+)\s*\]/gi;
44const imgCache = new Map(); // resolved path or paste key -> { file, w, h } | "fail"
45const pastes = new Map(); // [Image #N] number -> { data, key }
46const pasteMisses = new Map(); // [Image #N] number -> when the transcript last lacked it
47const PASTE_RETRY_MS = 2000;
48
49function run($, cmd, stdin) {
50  return $.process.run(["/bin/sh", "-c", cmd], stdin === undefined ? undefined : { stdin });
51}
52
53function quote(p) {
54  return `'${String(p).replace(/'/g, `'\\''`)}'`;
55}
56
57// The file name an alt/label shows: last path segment, query/fragment dropped.
58function basename(p) {
59  const clean = String(p).replace(/[?#].*/, "");
60  const m = /([^/\\]+)$/.exec(clean);
61  return m ? m[1] : clean;
62}
63
64// Absolute, ~/ against HOME, or cwd-joined for relative references
65// (.. stays intact). A ~/ path with no HOME resolves to null: no guess.
66function resolvePath(token, cwd, home) {
67  const t = token.trim();
68  if (t.startsWith("/")) return t;
69  if (t.startsWith("~/")) return home ? resolvePath(t.slice(2), home) : null;
70  const parts = `${String(cwd ?? "").replace(/\/+$/, "")}/${t.replace(/^\/+/, "")}`.split("/");
71  const out = [];
72  for (const p of parts) {
73    if (p === "" || p === ".") continue;
74    if (p === ".." && out.length && out[out.length - 1] !== "..") out.pop();
75    else out.push(p);
76  }
77  return `/${out.join("/")}`;
78}
79
80function hash(s) {
81  let h = 5381;
82  for (const c of s) h = ((h << 5) + h + c.charCodeAt(0)) | 0;
83  return (h >>> 0).toString(36);
84}
85
86// Run sh, then read the PNG's size; { file, w, h } when both worked, else null.
87async function decodeWith($, sh, png, stdin) {
88  try {
89    const r = await run($, `${sh} && sips -g pixelWidth -g pixelHeight ${quote(png)}`, stdin);
90    const w = /pixelWidth:\s*(\d+)/.exec(r?.stdout ?? "");
91    const h = /pixelHeight:\s*(\d+)/.exec(r?.stdout ?? "");
92    if ((r?.exitCode ?? 1) === 0 && w && h) return { file: png, w: parseInt(w[1], 10), h: parseInt(h[1], 10) };
93  } catch {}
94  return null;
95}
96
97// Decode absPath to a bounded PNG the terminal reads by name. A missing or
98// non-image file fails sips, so we never render a broken box.
99async function decode($, absPath) {
100  if (imgCache.has(absPath)) return imgCache.get(absPath);
101  const png = `${TMP_DIR}/${hash(absPath)}.png`;
102  // -Z bounds the longest side and preserves aspect (-z would resample to an
103  // exact WxH box and distort every picture into a square).
104  const result = await decodeWith(
105    $,
106    `mkdir -p ${quote(TMP_DIR)} && sips ${quote(absPath)} -Z 800 --out ${quote(png)} 2>/dev/null`,
107    png,
108  );
109  imgCache.set(absPath, result || "fail");
110  return result || "fail";
111}
112
113// Decode a pasted picture (base64 from the transcript) the same way. A paste
114// is the person's own content, so it goes to a directory only they can read
115// ($HOME/.cache/image-preview, mode 700, refused if not theirs or a symlink),
116// never the shared /tmp one; no HOME, no thumbnail.
117async function decodePaste($, paste, home) {
118  const { key } = paste;
119  if (imgCache.has(key)) return imgCache.get(key);
120  if (!home) return "fail";
121  const dir = `${home}/.cache/image-preview`;
122  const name = `${dir}/paste-${key}`;
123  const png = `${name}.png`;
124  const result = await decodeWith(
125    $,
126    `umask 077 && mkdir -p ${quote(dir)} && [ -O ${quote(dir)} ] && [ ! -L ${quote(dir)} ] && chmod 700 ${quote(dir)} && ` +
127      `/usr/bin/base64 -D -o ${quote(name)}.src && ` +
128      `sips ${quote(`${name}.src`)} -s format png -Z 800 --out ${quote(png)} >/dev/null 2>&1; ` +
129      `s=$?; rm -f ${quote(`${name}.src`)}; [ $s -eq 0 ]`,
130    png,
131    paste.data,
132  );
133  imgCache.set(key, result || "fail");
134  return result || "fail";
135}
136
137// Aspect-fill: the block's cell aspect matches the picture's pixel aspect, so
138// it is never stretched. A block of c x r cells is (c*16)px wide by (r*34)px
139// tall, so undistorted means c/r == (W/H) * 34/16 — `ratio` columns per row.
140// Thumbnails are height-bound: start from the row cap.
141function fit(W, H, maxColumns, maxRows) {
142  if (!W || !H || W <= 0 || H <= 0) return { columns: 1, rows: 1 };
143  const ratio = (W / H) * CELL_H_OVER_CELL_W;
144  let rows = Math.max(1, Math.floor(maxRows));
145  let columns = Math.round(rows * ratio);
146  if (columns > maxColumns) {
147    columns = Math.max(1, Math.floor(maxColumns));
148    rows = Math.max(1, Math.round(columns / ratio));
149  }
150  return { columns, rows };
151}
152
153// The image references in text, in document order: real paths and paste
154// tokens alike. The row's text is never rewritten, so nothing here carries the
155// surrounding prose.
156function imageRefs(text, cwd, home) {
157  if (!text || typeof text !== "string") return [];
158  const refs = [];
159  const quoted = []; // [start, end) of each quoted path, which PATH_RE skips
160  let m;
161  QUOTED_RE.lastIndex = 0;
162  while ((m = QUOTED_RE.exec(text))) {
163    // Several paths in one quoted span are prose, not one path: leave them
164    // to PATH_RE.
165    if (/\.(?:png|jpg|jpeg|gif|webp|bmp|tif(?:f)?|heic|heif|ico|svg)\s/i.test(m[2])) continue;
166    quoted.push([m.index, m.index + m[0].length]);
167    const abs = resolvePath(m[2], cwd, home);
168    if (abs) refs.push({ at: m.index, abs, label: basename(m[2]) });
169  }
170  PATH_RE.lastIndex = 0;
171  while ((m = PATH_RE.exec(text))) {
172    const at = m.index;
173    if (quoted.some(([s, e]) => at >= s && at < e)) continue;
174    const path = m[1].replace(/\\(.)/g, "$1");
175    const abs = resolvePath(path, cwd, home);
176    if (abs) refs.push({ at, abs, label: basename(path) });
177  }
178  REF_RE.lastIndex = 0;
179  while ((m = REF_RE.exec(text))) {
180    refs.push({ at: m.index, abs: null, num: parseInt(m[1], 10), label: `[Image #${m[1]}]` });
181  }
182  return refs.sort((a, b) => a.at - b.at);
183}
184
185// Learn the pictures behind [Image #N] tokens from the transcript: in a user
186// message the engine puts one base64 image block per token, in token order.
187// A message whose tokens and images do not pair up one to one (a token typed
188// by hand) teaches nothing, so a token never draws someone else's picture.
189async function learnPastes($) {
190  let messages;
191  try {
192    messages = await $.session.messages({ as: "api" });
193  } catch {
194    return;
195  }
196  if (!Array.isArray(messages)) return;
197  for (const msg of messages) {
198    if (msg?.role !== "user" || !Array.isArray(msg.content)) continue;
199    const text = msg.content.filter((b) => b?.type === "text").map((b) => b.text ?? "").join("\n");
200    const images = msg.content.filter((b) => b?.type === "image" && b.source?.type === "base64" && b.source.data);
201    const nums = [...text.matchAll(REF_RE)].map((t) => parseInt(t[1], 10));
202    if (!nums.length || nums.length !== images.length) continue;
203    nums.forEach((n, i) => {
204      if (pastes.has(n)) return;
205      const data = images[i].source.data;
206      pastes.set(n, { data, key: `${hash(data)}-${data.length}` });
207    });
208  }
209}
210
211// One thumbnail per resolvable reference, in document order.
212async function collectThumbs($, text, C) {
213  const cwd = (await $.session.cwd().catch(() => "/")).trim();
214  const home = await $.env.get("HOME").catch(() => undefined);
215  const refs = imageRefs(text, cwd, home);
216  // Read the transcript only for a paste not yet learned, and retry one it
217  // lacked (a token typed by hand, a row drawn before its message is stored)
218  // at most every PASTE_RETRY_MS, not on every redraw.
219  const now = Date.now();
220  const missing = refs.filter((r) => r.abs === null && !pastes.has(r.num));
221  if (missing.some((r) => !(now - (pasteMisses.get(r.num) ?? -Infinity) < PASTE_RETRY_MS))) {
222    await learnPastes($);
223    for (const r of missing) if (!pastes.has(r.num)) pasteMisses.set(r.num, now);
224  }
225  const nodes = [];
226  for (const ref of refs) {
227    let decoded;
228    if (ref.abs !== null) decoded = await decode($, ref.abs);
229    else if (pastes.has(ref.num)) decoded = await decodePaste($, pastes.get(ref.num), home);
230    if (!decoded || decoded === "fail") continue;
231    const path = decoded.file;
232    const sized = fit(decoded.w, decoded.h, C.maxColumns, MAX_ROWS);
233    nodes.push(
234      C.Box({
235        width: "100%",
236        alignItems: "center",
237        paddingY: 1,
238        children: [
239          C.Image({
240            key: hash(path),
241            source: { file: decoded.file, format: "png" },
242            columns: sized.columns,
243            rows: sized.rows,
244            alt: ref.label,
245          }),
246        ],
247      }),
248    );
249  }
250  return nodes;
251}
252
253// The mod's state: { render: boolean } — defaults to true. Some render passes read state before it is available (the raw
254// answer comes back { version } with no value), which would flicker the
255// thumbnails; remember the last successfully read state and fall back to it.
256let lastState = { render: true };
257
258async function loadState($) {
259  try {
260    const { value } = await $.state.get(NS);
261    const st = value?.state;
262    if (st && typeof st === "object") lastState = { render: st.render !== false };
263  } catch {}
264  return lastState;
265}
266
267function saveState($, state) {
268  lastState = state; // immediate consistency; the next read re-confirms
269  return $.state.set(NS, { state });
270}
271
272// Shared render body for both row types: the engine's own block, untouched —
273// identical to a no-image row — with the thumbnails appended below it.
274async function renderRow($, e, next) {
275  const state = await loadState($);
276  if (!state.render) return next(e); // off: a complete no-op
277
278  const props = e?.props ?? {};
279  const text = props.text;
280  if (typeof text !== "string") return next(e);
281
282  const maxColumns = e?.viewport?.columns ?? props.bodyColumns ?? 60;
283  const C = { ...$.ui.resolve(e), maxColumns };
284  const nodes = await collectThumbs($, text, C);
285  if (!nodes.length) return next(e);
286
287  const body = await next(e);
288  const bodyNode = typeof body === "string" ? C.Text({ children: body }) : body;
289  return C.Box({ flexDirection: "column", children: [bodyNode, ...nodes] });
290}
291
292export function register(on) {
293  on("ui.render", { component: "UserMessage" }, async ($, e, next) => {
294    if (e?.props?.origin?.kind !== "composer") return next(e);
295    return renderRow($, e, next);
296  });
297
298  on("ui.render", { component: "AssistantMessage" }, ($, e, next) => renderRow($, e, next));
299
300  // /image-preview on|off toggles thumbnails for the session.
301  on("session.start", async ($, e, next) => {
302    const r = await next(e);
303    try {
304      await $.command.register({
305        name: "image-preview",
306        description: "Image previews: thumbnails on|off for this session",
307        argumentHint: "[on|off]",
308      });
309    } catch {
310      // already registered after a hot reload
311    }
312    return r;
313  });
314
315  on("command.run", async ($, e, next) => {
316    if (e?.command !== "image-preview") return next(e);
317    const arg = (e?.args ?? "").trim();
318    let render;
319    if (/^(on|off)$/.test(arg)) {
320      render = arg === "on";
321      await saveState($, { render });
322    } else {
323      ({ render } = await loadState($));
324    }
325    return {
326      text: render
327        ? "Thumbnails ON: image paths and pasted images draw inline. /image-preview off to switch."
328        : "Thumbnails OFF: rows draw as without the mod. /image-preview on to switch.",
329    };
330  });
331}
332

1declare module "claude-code" {
2  interface PluginState {
3    "image-preview": {
4      state: {
5        /** Whether thumbnails draw this session; /image-preview on|off sets
6         * it, and absent means on. */
7        render?: boolean;
8      };
9    };
10  }
11}
12